5 #include <linux/module.h>
6 #include <linux/moduleparam.h>
7 #include <linux/init.h>
8 #include <linux/sched.h>
9 #include <linux/kernel.h>
10 #include <linux/slab.h>
12 #include <linux/errno.h>
13 #include <linux/timer.h>
14 #include <linux/types.h>
15 #include <linux/vmalloc.h>
16 #include <linux/genhd.h>
17 #include <linux/blkdev.h>
18 #include <linux/bio.h>
23 #define XSEGBD_MINORS 1
25 MODULE_DESCRIPTION("xsegbd");
26 MODULE_AUTHOR("XSEG");
27 MODULE_LICENSE("GPL");
29 static long sector_size = 100000;
30 static long blksize = 512;
32 static char name[XSEGBD_VOLUME_NAMELEN] = "xsegbd";
33 static char spec[256] = "xsegdev:xsegbd:4:512:64:1024:12";
34 static int src_portno = 0, dst_portno = 1, nr_requests = 128;
36 module_param(sector_size, long, 0644);
37 module_param(blksize, long, 0644);
38 module_param(major, int, 0644);
39 module_param(src_portno, int, 0644);
40 module_param(dst_portno, int, 0644);
41 module_param(nr_requests, int, 0644);
42 module_param_string(name, name, sizeof(name), 0644);
43 module_param_string(spec, spec, sizeof(spec), 0644);
45 static volatile int count;
46 struct semaphore xsegbd_lock;
47 static struct xsegbd xsegbd;
50 /* ********************* */
51 /* ** XSEG Operations ** */
52 /* ********************* */
54 static void *xsegdev_malloc(uint64_t size)
56 return kmalloc((size_t)size, GFP_KERNEL);
59 static void *xsegdev_realloc(void *mem, uint64_t size)
61 return krealloc(mem, (size_t)size, GFP_KERNEL);
64 static void xsegdev_mfree(void *ptr)
69 static long xsegdev_allocate(const char *name, uint64_t size)
72 struct xsegdev *xsegdev = xsegdev_get(0);
74 r = IS_ERR(xsegdev) ? PTR_ERR(xsegdev) : 0;
76 XSEGLOG("cannot acquire xsegdev");
80 if (xsegdev->segment) {
81 XSEGLOG("destroying existing xsegdev segment");
82 r = xsegdev_destroy_segment(xsegdev);
87 XSEGLOG("creating xsegdev segment size %llu", size);
88 r = xsegdev_create_segment(xsegdev, size, 1);
92 xsegdev->segsize = size;
100 static long xsegdev_deallocate(const char *name)
102 struct xsegdev *xsegdev = xsegdev_get(0);
103 int r = IS_ERR(xsegdev) ? PTR_ERR(xsegdev) : 0;
107 clear_bit(XSEGDEV_RESERVED, &xsegdev->flags);
108 XSEGLOG("destroying segment");
109 r = xsegdev_destroy_segment(xsegdev);
111 XSEGLOG(" ...failed");
112 xsegdev_put(xsegdev);
116 static long xseg_callback(void *arg);
118 static void *xsegdev_map(const char *name, uint64_t size)
120 struct xseg *xseg = NULL;
121 struct xsegdev *dev = xsegdev_get(0);
123 r = IS_ERR(dev) ? PTR_ERR(dev) : 0;
130 if (size > dev->segsize)
133 if (dev->callback) /* in use */
136 dev->callback = xseg_callback;
137 dev->callarg = &xsegbd;
138 xseg = (void *)dev->segment;
146 static void xsegdev_unmap(void *ptr, uint64_t size)
148 struct xsegdev *xsegdev = xsegdev_get(0);
149 int r = IS_ERR(xsegdev) ? PTR_ERR(xsegdev) : 0;
153 xsegdev->callarg = NULL;
154 xsegdev->callback = NULL;
155 xsegdev_put(xsegdev);
158 static struct xseg_type xseg_xsegdev = {
159 /* xseg operations */
161 .malloc = xsegdev_malloc,
162 .realloc = xsegdev_realloc,
163 .mfree = xsegdev_mfree,
164 .allocate = xsegdev_allocate,
165 .deallocate = xsegdev_deallocate,
167 .unmap = xsegdev_unmap
173 static int posix_signal_init(void)
178 static void posix_signal_quit(void) { }
180 static int posix_prepare_wait(struct xseg_port *port)
185 static int posix_cancel_wait(struct xseg_port *port)
190 static int posix_wait_signal(struct xseg_port *port, uint32_t timeout)
195 static int posix_signal(struct xseg_port *port)
198 struct task_struct *task;
202 pid = find_vpid((pid_t)port->waitcue);
205 task = pid_task(pid, PIDTYPE_PID);
209 ret = send_sig(SIGIO, task, 1);
215 static void *posix_malloc(uint64_t size)
220 static void *posix_realloc(void *mem, uint64_t size)
225 static void posix_mfree(void *mem) { }
227 static struct xseg_peer xseg_peer_posix = {
228 /* xseg signal operations */
230 .signal_init = posix_signal_init,
231 .signal_quit = posix_signal_quit,
232 .cancel_wait = posix_cancel_wait,
233 .prepare_wait = posix_prepare_wait,
234 .wait_signal = posix_wait_signal,
235 .signal = posix_signal,
236 .malloc = posix_malloc,
237 .realloc = posix_realloc,
244 static int xsegdev_signal_init(void)
249 static void xsegdev_signal_quit(void) { }
251 static int xsegdev_prepare_wait(struct xseg_port *port)
256 static int xsegdev_cancel_wait(struct xseg_port *port)
261 static int xsegdev_wait_signal(struct xseg_port *port, uint32_t timeout)
266 static int xsegdev_signal(struct xseg_port *port)
271 static struct xseg_peer xseg_peer_xsegdev = {
272 /* xseg signal operations */
274 .signal_init = xsegdev_signal_init,
275 .signal_quit = xsegdev_signal_quit,
276 .cancel_wait = xsegdev_cancel_wait,
277 .prepare_wait = xsegdev_prepare_wait,
278 .wait_signal = xsegdev_wait_signal,
279 .signal = xsegdev_signal,
280 .malloc = xsegdev_malloc,
281 .realloc = xsegdev_realloc,
282 .mfree = xsegdev_mfree
288 /* ************************* */
289 /* ** XSEG Initialization ** */
290 /* ************************* */
292 int xsegbd_xseg_init(struct xsegbd *dev)
294 struct xseg_port *xport;
298 strncpy(dev->name, name, XSEGBD_VOLUME_NAMELEN);
300 XSEGLOG("registering xseg types");
301 dev->namesize = strlen(dev->name);
302 r = xseg_register_type(&xseg_xsegdev);
306 r = xseg_register_peer(&xseg_peer_posix);
310 r = xseg_register_peer(&xseg_peer_xsegdev);
314 r = xseg_initialize("xsegdev");
316 XSEGLOG("cannot initialize 'xsegdev' peer");
320 r = xseg_parse_spec(spec, &dev->config);
324 if (strncmp(dev->config.type, "xsegdev", 16))
325 XSEGLOG("WARNING: unexpected segment type '%s' vs 'xsegdev'",
328 XSEGLOG("creating segment");
329 r = xseg_create(&dev->config);
331 XSEGLOG("cannot create segment");
335 XSEGLOG("joining segment");
336 dev->xseg = xseg_join("xsegdev", "xsegbd");
338 XSEGLOG("cannot join segment");
343 XSEGLOG("binding to source port %u (destination %u)",
344 src_portno, dst_portno);
345 xport = xseg_bind_port(dev->xseg, src_portno);
347 XSEGLOG("cannot bind to port");
352 dev->src_portno = xseg_portno(dev->xseg, xport);
353 dev->dst_portno = dst_portno;
355 if (nr_requests > dev->xseg->config.nr_requests)
356 nr_requests = dev->xseg->config.nr_requests;
358 if (xseg_alloc_requests(dev->xseg, src_portno, nr_requests)) {
359 XSEGLOG("cannot allocate requests");
367 xseg_unregister_peer(xseg_peer_xsegdev.name);
369 xseg_unregister_peer(xseg_peer_posix.name);
371 xseg_unregister_type(xseg_xsegdev.name);
376 int xsegbd_xseg_quit(struct xsegbd *dev)
378 xseg_destroy(dev->xseg);
384 /* ***************************** */
385 /* ** Block Device Operations ** */
386 /* ***************************** */
388 static int xsegbd_open(struct block_device *bdev, fmode_t mode)
390 int ret = down_interruptible(&xsegbd_lock);
398 static int xsegbd_release(struct gendisk *gd, fmode_t mode)
400 int ret = down_interruptible(&xsegbd_lock);
408 static int xsegbd_ioctl(struct block_device *bdev, fmode_t mode,
409 unsigned int cmd, unsigned long arg)
414 static const struct block_device_operations xsegbd_ops = {
415 .owner = THIS_MODULE,
417 .release = xsegbd_release,
418 .ioctl = xsegbd_ioctl
422 /* *************************** */
423 /* ** Device Initialization ** */
424 /* *************************** */
426 static void xseg_request_fn(struct request_queue *rq);
428 static loff_t xsegbd_get_size(struct xsegbd *dev)
430 struct xseg_request *xreq;
435 xseg_prepare_wait(dev->xseg, dev->src_portno);
437 if ((xreq = xseg_get_request(dev->xseg, dev->src_portno))) {
438 xseg_cancel_wait(dev->xseg, dev->src_portno);
440 datasize = sizeof(loff_t);
441 BUG_ON(xreq->buffersize - dev->namesize < datasize);
442 BUG_ON(xseg_prep_request(xreq, dev->namesize, datasize));
444 name = XSEG_TAKE_PTR(xreq->name, dev->xseg->segment);
445 strncpy(name, dev->name, dev->namesize);
446 xreq->size = datasize;
451 BUG_ON(xseg_submit(dev->xseg, dev->dst_portno, xreq) == NoSerial);
453 xseg_signal(dev->xseg, dev->dst_portno);
456 while (!(xreq = xseg_receive(dev->xseg, dev->src_portno))) ;
458 xseg_cancel_wait(dev->xseg, dev->src_portno);
459 while (!(xreq->state & XS_SERVED)) ;
461 data = XSEG_TAKE_PTR(xreq->data, dev->xseg->segment);
462 size = *((off_t *) data);
465 xseg_put_request(dev->xseg, dev->src_portno, xreq);
470 static int xsegbd_dev_init(struct xsegbd *dev, int id, sector_t size)
473 struct gendisk *disk;
475 spin_lock_init(&dev->lock);
478 dev->blk_queue = blk_alloc_queue(GFP_KERNEL);
482 blk_init_allocated_queue(dev->blk_queue, xseg_request_fn, &dev->lock);
483 dev->blk_queue->queuedata = dev;
485 blk_queue_flush(dev->blk_queue, REQ_FLUSH | REQ_FUA);
486 blk_queue_logical_block_size(dev->blk_queue, 512);
487 blk_queue_physical_block_size(dev->blk_queue, blksize);
488 blk_queue_bounce_limit(dev->blk_queue, BLK_BOUNCE_ANY);
489 /* we can handle any number of segments, BUT
490 * parts of the request may be available far sooner than others
491 * but we cannot complete them (unless we handle their bios directly).
493 blk_queue_max_segments(dev->blk_queue, 1);
494 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, dev->blk_queue);
496 /* vkoukis says we don't need partitions */
497 dev->gd = disk = alloc_disk(1);
502 disk->first_minor = id * XSEGBD_MINORS;
503 disk->fops = &xsegbd_ops;
504 disk->queue = dev->blk_queue;
505 disk->private_data = dev;
506 disk->flags |= GENHD_FL_SUPPRESS_PARTITION_INFO;
507 snprintf(disk->disk_name, 32, "xsegbd%c", 'a' + id);
509 ret = xsegbd_xseg_init(dev);
513 if (!xq_alloc_seq(&dev->blk_queue_pending, nr_requests, nr_requests))
516 dev->blk_req_pending = kmalloc(sizeof(struct request *) * nr_requests, GFP_KERNEL);
517 if (!dev->blk_req_pending)
518 goto out_free_pending;
520 dev->sectors = xsegbd_get_size(dev) / 512ULL;
521 set_capacity(disk, dev->sectors);
523 add_disk(disk); /* immediately activates the device */
529 xq_free(&dev->blk_queue_pending);
532 xsegbd_xseg_quit(dev);
538 blk_cleanup_queue(dev->blk_queue);
543 static int xsegbd_dev_destroy(struct xsegbd *dev)
545 xq_free(&dev->blk_queue_pending);
546 kfree(dev->blk_req_pending);
547 del_gendisk(dev->gd);
549 blk_cleanup_queue(dev->blk_queue);
550 xsegbd_xseg_quit(dev);
555 /* *************************** */
556 /* ** Module Initialization ** */
557 /* *************************** */
559 static int __init xsegbd_init(void)
563 sema_init(&xsegbd_lock, 1);
565 XSEGLOG("registering block device major %d", major);
566 ret = register_blkdev(major, XSEGBD_NAME);
568 XSEGLOG("cannot register block device!");
573 XSEGLOG("registered block device major %d", major);
575 XSEGLOG("initializing device");
576 ret = xsegbd_dev_init(&xsegbd, 0, sector_size);
578 XSEGLOG("cannot initialize device!");
582 XSEGLOG("initialization complete");
587 unregister_blkdev(major, XSEGBD_NAME);
591 static void __exit xsegbd_exit(void)
593 unregister_blkdev(major, XSEGBD_NAME);
595 xseg_disable_driver(xsegbd.xseg, "posix");
596 xseg_unregister_peer("posix");
597 xseg_disable_driver(xsegbd.xseg, "xsegdev");
598 xseg_unregister_peer("xsegdev");
600 xsegbd_dev_destroy(&xsegbd);
601 xseg_unregister_type("xsegdev");
604 module_init(xsegbd_init);
605 module_exit(xsegbd_exit);
608 /* ******************* */
609 /* ** Critical Path ** */
610 /* ******************* */
612 static void blk_to_xseg(struct xseg *xseg, struct xseg_request *xreq,
613 struct request *blkreq)
615 struct bio_vec *bvec;
616 struct req_iterator iter;
618 char *data = XSEG_TAKE_PTR(xreq->data, xseg->segment);
619 rq_for_each_segment(bvec, blkreq, iter) {
620 char *bdata = kmap_atomic(bvec->bv_page) + bvec->bv_offset;
621 memcpy(data + off, bdata, bvec->bv_len);
623 kunmap_atomic(bdata);
627 static void xseg_to_blk(struct xseg *xseg, struct xseg_request *xreq,
628 struct request *blkreq)
630 struct bio_vec *bvec;
631 struct req_iterator iter;
633 char *data = XSEG_TAKE_PTR(xreq->data, xseg->segment);
634 rq_for_each_segment(bvec, blkreq, iter) {
635 char *bdata = kmap_atomic(bvec->bv_page) + bvec->bv_offset;
636 memcpy(bdata, data + off, bvec->bv_len);
638 kunmap_atomic(bdata);
642 static void xseg_request_fn(struct request_queue *rq)
644 struct xseg_request *xreq;
645 struct xsegbd *dev = rq->queuedata;
646 struct request *blkreq;
652 xreq = xseg_get_request(dev->xseg, dev->src_portno);
656 blkreq = blk_fetch_request(rq);
660 if (blkreq->cmd_type != REQ_TYPE_FS) {
661 XSEGLOG("non-fs cmd_type: %u. *shrug*", blkreq->cmd_type);
662 __blk_end_request_all(blkreq, 0);
665 datasize = blk_rq_bytes(blkreq);
666 BUG_ON(xreq->buffersize - dev->namesize < datasize);
667 BUG_ON(xseg_prep_request(xreq, dev->namesize, datasize));
669 name = XSEG_TAKE_PTR(xreq->name, dev->xseg->segment);
670 strncpy(name, dev->name, dev->namesize);
671 blkreq_idx = xq_pop_head(&dev->blk_queue_pending);
672 BUG_ON(blkreq_idx == None);
673 /* WARN_ON(dev->blk_req_pending[blkreq_idx] */
674 dev->blk_req_pending[blkreq_idx] = blkreq;
675 xreq->priv = (void *)(unsigned long)blkreq_idx;
676 xreq->size = datasize;
677 xreq->offset = blk_rq_pos(blkreq) << 9;
679 if (xreq->offset >= (sector_size << 9))
680 XSEGLOG("sector offset: %lu > %lu, flush:%u, fua:%u",
681 blk_rq_pos(blkreq), sector_size,
682 blkreq->cmd_flags & REQ_FLUSH,
683 blkreq->cmd_flags & REQ_FUA);
686 if (blkreq->cmd_flags & REQ_FLUSH)
687 xreq->flags |= XF_FLUSH;
689 if (blkreq->cmd_flags & REQ_FUA)
690 xreq->flags |= XF_FUA;
692 if (rq_data_dir(blkreq)) {
693 /* unlock for data transfers? */
694 blk_to_xseg(dev->xseg, xreq, blkreq);
700 BUG_ON(xseg_submit(dev->xseg, dev->dst_portno, xreq) == NoSerial);
704 xseg_put_request(dev->xseg, dev->src_portno, xreq);
707 static long xseg_callback(void *arg)
709 struct xsegbd *dev = arg;
710 struct xseg_request *xreq;
711 struct request *blkreq;
717 xreq = xseg_receive(dev->xseg, dev->src_portno);
721 /* we rely upon our peers to not have touched ->priv */
722 blkreq_idx = (xqindex)(unsigned long)xreq->priv;
723 if (blkreq_idx < 0 || blkreq_idx >= nr_requests) {
724 XSEGLOG("invalid request index: %u! Ignoring.", blkreq_idx);
728 blkreq = dev->blk_req_pending[blkreq_idx];
729 /* WARN_ON(!blkreq); */
732 if (!(xreq->state & XS_SERVED))
735 if (xreq->serviced != blk_rq_bytes(blkreq))
738 /* unlock for data transfer? */
739 if (!rq_data_dir(blkreq))
740 xseg_to_blk(dev->xseg, xreq, blkreq);
744 blk_end_request_all(blkreq, err);
745 xq_append_head(&dev->blk_queue_pending, blkreq_idx);
747 xseg_put_request(dev->xseg, xreq->portno, xreq);
750 spin_lock_irqsave(&dev->lock, flags);
751 xseg_request_fn(dev->blk_queue);
752 spin_unlock_irqrestore(&dev->lock, flags);