Various fixes to make xseg functional again

 * xseg:
	- Make sure the allocated xseg struct in xseg_join is initialized
	  correctly
	- Initial implementation for xseg_leave()
	- Add __lock_domain in xseg_destroy()

 * driver api:
	- Add a struct xseg * argument to xseg_operations->map(). It's
	  passed to the driver when a peer joins the segment, and it's
	  used by xseg_segdev kernel driver (for the initialization of
	  xsegments[] array).

 * drivers/kernel/xseg_segdev:
    - Use struct xseg * segdev_map() to initialize the xsegments[]
	  array.
	- Some minor fixeds

 * sys/segdev:
    - Fix wait_for_event_interruptible condition (this fixes an issue
	  with xseg-tool not being able to destroy segdev segments).
	- Correct the use of segdev_get/put() in various places.

 * build system:
    - Add distclean target, which cleans lib/user and lib/kernel dirs.
	- Fix the clean target in sys/user dir.
	- Temp exclude of the vlmc peer (it doesn't compile)

 * tools:
    - Modify the helper/setup/vlmc scripts, to work with the new
	  repo/code layout.

 * xsegbd:
    - Minor fix in xseg_callback() (xseg_receive() wasn't fixed after
	  the refactoring)

diff --git a/xseg/Makefile b/xseg/Makefile
index 28c6fd6..bc40979 100644 (file)
--- a/xseg/Makefile
+++ b/xseg/Makefile
@@ -1,4 +1,4 @@
-.PHONY: default clean target-list
+.PHONY: default clean target-list distclean
 
 TARGET=default
 include ./base.mk
@@ -37,3 +37,6 @@ $(XSEG_CLEAN_TARGETS):
        rm -f ./config.mk
        @echo " "
 
+distclean:
+       rm -f lib/user/*.so* lib/user/*.a
+       rm -f lib/kernel/*.ko

diff --git a/xseg/drivers/kernel/xseg_segdev.c b/xseg/drivers/kernel/xseg_segdev.c
index f0bfb12..5b52c89 100644 (file)
--- a/xseg/drivers/kernel/xseg_segdev.c
+++ b/xseg/drivers/kernel/xseg_segdev.c
@@ -95,11 +95,12 @@ static long segdev_deallocate(const char *name)
        return r;
 }
 
-static void *segdev_map(const char *name, uint64_t size)
+static void *segdev_map(const char *name, uint64_t size, struct xseg *seg)
 {
        struct xseg *xseg = NULL;
        /* map() holds a reference to the segment */
        struct segdev *dev = segdev_get(0);
+       struct segpriv *priv;
        int r;
        r = IS_ERR(dev) ? PTR_ERR(dev) : 0;
        if (r)
@@ -111,6 +112,13 @@ static void *segdev_map(const char *name, uint64_t size)
        if (size > dev->segsize)
                goto out;
 
+       priv = dev->priv;
+       if (priv->segno >= nr_xsegments)
+               goto out;
+
+       if (seg)
+               xsegments[priv->segno] = seg;
+
        xseg = (void *)dev->segment;
 out:
        return xseg;
@@ -119,12 +127,22 @@ out:
 static void segdev_unmap(void *ptr, uint64_t size)
 {
        struct segdev *segdev = segdev_get(0);
+       struct segpriv *priv;
        int r = IS_ERR(segdev) ? PTR_ERR(segdev) : 0;
        if (r)
                return;
 
+       priv = segdev->priv;
+       if (priv->segno >= nr_xsegments)
+               goto out;
+
+       xsegments[priv->segno] = NULL;
+
+out:
        /* unmap() releases the reference taken by map() */
        segdev_put(segdev);
+
+       segdev_put(segdev);
 }
 
 static void segdev_callback(struct segdev *dev)
@@ -133,7 +151,7 @@ static void segdev_callback(struct segdev *dev)
        struct segpriv *priv = dev->priv;
        struct xseg_private *xpriv;
        uint32_t portno;
-       if (priv->segno > nr_xsegments)
+       if (priv->segno >= nr_xsegments)
                return;
 
        if (dev->buffer_index != sizeof(uint32_t)) {
@@ -141,11 +159,12 @@ static void segdev_callback(struct segdev *dev)
                return;
        }
 
-       portno = *(uint32_t *)dev->buffer;
        xseg = xsegments[priv->segno];
        xpriv = xseg->priv;
-       if (xpriv->wakeup)
+       if (xpriv->wakeup) {
+               portno = *(uint32_t *)dev->buffer;
                xpriv->wakeup(xseg, portno);
+       }
 }
 
 static struct xseg_type xseg_segdev = {
@@ -182,6 +201,7 @@ static int segdev_signal_init(void)
        segdev->priv = segpriv;
        r = 0;
 out:
+       segdev_put(segdev);
        return r;
 }
 
@@ -192,6 +212,8 @@ static void segdev_signal_quit(void)
        xsegments[0] = NULL;
        if (!r)
                segdev->callback = NULL;
+
+       segdev_put(segdev);
        return;
 }
 

diff --git a/xseg/drivers/user/xseg_posix.c b/xseg/drivers/user/xseg_posix.c
index 554487f..0d71e09 100644 (file)
--- a/xseg/drivers/user/xseg_posix.c
+++ b/xseg/drivers/user/xseg_posix.c
@@ -52,10 +52,14 @@ static long posix_deallocate(const char *name)
        return shm_unlink(name);
 }
 
-static void *posix_map(const char *name, uint64_t size)
+static void *posix_map(const char *name, uint64_t size, struct xseg *seg)
 {
        struct xseg *xseg;
        int fd;
+
+       if (seg)
+               XSEGLOG("struct xseg * is not NULL. Ignoring...\n");
+
        fd = shm_open(name, O_RDWR, 0000);
        if (fd < 0) {
                XSEGLOG("Failed to open '%s' for mapping: %s\n",

diff --git a/xseg/drivers/user/xseg_segdev.c b/xseg/drivers/user/xseg_segdev.c
index dae521c..b6d24aa 100644 (file)
--- a/xseg/drivers/user/xseg_segdev.c
+++ b/xseg/drivers/user/xseg_segdev.c
@@ -99,10 +99,14 @@ static long segdev_deallocate(const char *name)
        return 0;
 }
 
-static void *segdev_map(const char *name, uint64_t size)
+static void *segdev_map(const char *name, uint64_t size, struct xseg *seg)
 {
        struct xseg *xseg;
        int fd;
+
+       if (seg)
+               XSEGLOG("struct xseg * not NULL. Ignoring...\n");
+
        fd = opendev();
        if (fd < 0)
                return NULL;
@@ -169,8 +173,7 @@ static int segdev_wait_signal(struct xseg *xseg, uint32_t timeout)
 
 static int segdev_signal(struct xseg *xseg, uint32_t portno)
 {
-       struct xseg_port *port = &xseg->ports[portno];
-       return write(opendev(), &port, sizeof(port));
+       return write(opendev(), &portno, sizeof(portno));
 }
 
 static void *segdev_malloc(uint64_t size)

diff --git a/xseg/peers/kernel/xsegbd.c b/xseg/peers/kernel/xsegbd.c
index fd611b9..120ac59 100644 (file)
--- a/xseg/peers/kernel/xsegbd.c
+++ b/xseg/peers/kernel/xsegbd.c
@@ -478,7 +478,7 @@ static void xseg_callback(struct xseg *xseg, uint32_t portno)
        int err;
 
        for (;;) {
-               xreq = xseg_receive(xseg, xsegbd_dev->src_portno);
+               xreq = xseg_receive(xseg, portno);
                if (!xreq)
                        break;
 

diff --git a/xseg/peers/user/Makefile b/xseg/peers/user/Makefile
index 4383e98..d936400 100644 (file)
--- a/xseg/peers/user/Makefile
+++ b/xseg/peers/user/Makefile
@@ -4,7 +4,7 @@ include $(XSEG_HOME)/base.mk
 
 default: all
 
-all: blockd filed vlmcd xseg
+all: blockd filed xseg
 
 filed: filed.c $(BASE)/xseg/xseg.h
        $(CC) $(CFLAGS) -o $@ $< $(INC) -L$(LIB) -lxseg -lpthread

diff --git a/xseg/peers/user/xseg-tool.c b/xseg/peers/user/xseg-tool.c
index af37d70..f744d34 100644 (file)
--- a/xseg/peers/user/xseg-tool.c
+++ b/xseg/peers/user/xseg-tool.c
@@ -674,6 +674,7 @@ int cmd_destroy(void)
 {
        if (!xseg && cmd_join())
                return -1;
+       xseg_leave(xseg);
        xseg_destroy(xseg);
        xseg = NULL;
        fprintf(stderr, "Segment destroyed.\n");

diff --git a/xseg/sys/kernel/segdev.c b/xseg/sys/kernel/segdev.c
index 6eec912..ee06869 100644 (file)
--- a/xseg/sys/kernel/segdev.c
+++ b/xseg/sys/kernel/segdev.c
@@ -82,7 +82,7 @@ int segdev_destroy_segment(struct segdev *dev)
                goto out_unlock;
 
        clear_bit(SEGDEV_READY, &dev->flags);
-       ret = wait_event_interruptible(dev->wq, atomic_read(&dev->usercount) < 1);
+       ret = wait_event_interruptible(dev->wq, atomic_read(&dev->usercount) <= 1);
        if (ret)
                goto out_unlock;
 

diff --git a/xseg/sys/user/Makefile b/xseg/sys/user/Makefile
index 3c68b59..111acfc 100644 (file)
--- a/xseg/sys/user/Makefile
+++ b/xseg/sys/user/Makefile
@@ -67,7 +67,6 @@ libxseg.a: xseg.o xq/xq.o drivers _initialize.o
 clean:
        make -C xq clean
        rm -f _initialize.c _initialize.o
-       rm -f xseg.o xseg.pic.o
-       rm -f libxseg.a
+       rm -f xseg.o xseg.pic.o xseg_user.o
+       rm -f libxseg.a libxseg.map
        rm -f libxseg.so libxseg.so.$(MAJOR) libxseg.so.$(MAJOR).$(MINOR)
-

diff --git a/xseg/tools/helpers.sh b/xseg/tools/helpers.sh
index 9bcda3c..14f7bbd 100755 (executable)
--- a/xseg/tools/helpers.sh
+++ b/xseg/tools/helpers.sh
@@ -62,6 +62,7 @@ function load_all {
        load_module "segdev"
        load_module "xseg_segdev"
        mk_chardev
+        "${XSEG_HOME}/peers/user/xseg" "${SPEC}" create
        load_module "xsegbd" "spec=$SPEC"
 }
 
@@ -70,11 +71,11 @@ function load_all {
 # @param $1            target/volume name
 # @param $2            xseg port
 function spawn_blockd {
-       "${XSEG_HOME}peers/blockd" "$1" -p "$2" -g "$SPEC" -n ${NR_OPS} &> "${BLOCKD_LOGS}/$1" || fail "blockd"
+       "${XSEG_HOME}peers/user/blockd" "$1" -p "$2" -g "$SPEC" -n ${NR_OPS} &> "${BLOCKD_LOGS}/$1" || fail "blockd"
 }
 
 function spawn_filed {
-       "${XSEG_HOME}peers/filed" "$1" -p "$2" -g "${SPEC}" -n ${NR_OPS} &> "${BLOCKD_LOGS}/filed-${HOSTNAME}" &
+       "${XSEG_HOME}peers/user/filed" "$1" -p "$2" -g "${SPEC}" -n ${NR_OPS} &> "${BLOCKD_LOGS}/filed-${HOSTNAME}" &
 }
 
 # map_volume - Map a volume to an xsegbd device

diff --git a/xseg/tools/vlmc-blockd.py b/xseg/tools/vlmc-blockd.py
index 6b7ed8c..c5c414e 100755 (executable)
--- a/xseg/tools/vlmc-blockd.py
+++ b/xseg/tools/vlmc-blockd.py
@@ -30,7 +30,7 @@ def vlmc_map(args):
         old_dir = os.getcwd()
         os.chdir(IMAGES)
         f = os.open(BLOCKD_LOGS +  name, os.O_CREAT | os.O_WRONLY)
-        r = subprocess.Popen([XSEG_HOME + "peers/blockd", name, "-p", str(port),
+        r = subprocess.Popen([XSEG_HOME + "peers/user/blockd", name, "-p", str(port),
         "-g", SPEC, "-n", str(NR_OPS)], stdout=f, stderr=f)
 
         os.chdir(IMAGES)

diff --git a/xseg/tools/vlmc_shared.py b/xseg/tools/vlmc_shared.py
index 312735f..f986220 100644 (file)
--- a/xseg/tools/vlmc_shared.py
+++ b/xseg/tools/vlmc_shared.py
@@ -9,6 +9,7 @@ DEVICE_PREFIX="/dev/xsegbd"
 BLOCKD_LOGS="/root/archip_logs/"
 FILED_PORT=0
 NR_OPS=16
+REQS=512
 
 def vlmc_list(args):
     print "name\t\t\t\tsize"

diff --git a/xseg/xseg/xseg.c b/xseg/xseg/xseg.c
index 13f108d..5f0da81 100644 (file)
--- a/xseg/xseg/xseg.c
+++ b/xseg/xseg/xseg.c
@@ -538,7 +538,7 @@ int xseg_create(struct xseg_config *cfg)
                goto out_err;
        }
 
-       xseg = xops->map(cfg->name, size);
+       xseg = xops->map(cfg->name, size, NULL);
        if (!xseg) {
                XSEGLOG("cannot map segment!\n");
                goto out_deallocate;
@@ -561,14 +561,19 @@ out_err:
 
 void xseg_destroy(struct xseg *xseg)
 {
-       struct xseg_type *type = __find_or_load_type(xseg->config.type);
+       struct xseg_type *type;
+
+       __lock_domain();
+       type = __find_or_load_type(xseg->config.type);
        if (!type) {
                XSEGLOG("no segment type '%s'\n", xseg->config.type);
-               return;
+               goto out;
        }
 
        /* should destroy() leave() first? */
        type->ops.deallocate(xseg->config.name);
+out:
+       __unlock_domain();
 }
 
 static int pointer_ok( unsigned long ptr,
@@ -652,7 +657,7 @@ struct xseg *xseg_join(     char *segtypename,
                goto err_seg;
        }
 
-       __xseg = xops->map(segname, XSEG_MIN_PAGE_SIZE);
+       __xseg = xops->map(segname, XSEG_MIN_PAGE_SIZE, NULL);
        if (!__xseg) {
                XSEGLOG("Cannot map segment");
                goto err_priv;
@@ -662,7 +667,7 @@ struct xseg *xseg_join(     char *segtypename,
        /* XSEGLOG("joined segment of size: %lu\n", (unsigned long)size); */
        xops->unmap(__xseg, XSEG_MIN_PAGE_SIZE);
 
-       __xseg = xops->map(segname, size);
+       __xseg = xops->map(segname, size, xseg);
        if (!__xseg) {
                XSEGLOG("Cannot map segment");
                goto err_priv;
@@ -671,6 +676,7 @@ struct xseg *xseg_join(     char *segtypename,
        priv->segment_type = *segtype;
        priv->peer_type = *peertype;
        priv->wakeup = wakeup;
+       xseg->max_peer_types = __xseg->max_peer_types;
 
        priv->peer_types = pops->malloc(sizeof(void *) * xseg->max_peer_types);
        if (!priv->peer_types) {
@@ -679,6 +685,7 @@ struct xseg *xseg_join(     char *segtypename,
        }
        memset(priv->peer_types, 0, sizeof(void *) * xseg->max_peer_types);
 
+       xseg->priv = priv;
        xseg->config = __xseg->config;
        xseg->version = __xseg->version;
        xseg->requests = XSEG_TAKE_PTR(__xseg->requests, __xseg);
@@ -718,7 +725,21 @@ err:
        return NULL;
 }
 
-/* void xseg_leave(struct xseg *xseg) { at least free allocated memory } */
+void xseg_leave(struct xseg *xseg)
+{
+       struct xseg_type *type;
+
+       __lock_domain();
+       type = __find_or_load_type(xseg->config.type);
+       if (!type) {
+               XSEGLOG("no segment type '%s'\n", xseg->config.type);
+               __unlock_domain();
+               return;
+       }
+       __unlock_domain();
+
+       type->ops.unmap(xseg->segment, xseg->segment_size);
+}
 
 int xseg_prepare_wait(struct xseg *xseg, uint32_t portno)
 {

diff --git a/xseg/xseg/xseg.h b/xseg/xseg/xseg.h
index e6bc44d..c1a2f96 100644 (file)
--- a/xseg/xseg/xseg.h
+++ b/xseg/xseg/xseg.h
@@ -61,7 +61,7 @@ struct xseg_operations {
        void  (*mfree)(void *mem);
        long  (*allocate)(const char *name, uint64_t size);
        long  (*deallocate)(const char *name);
-       void *(*map)(const char *name, uint64_t size);
+       void *(*map)(const char *name, uint64_t size, struct xseg *seg);
        void  (*unmap)(void *xseg, uint64_t size);
 };
 

diff --git a/xseg/xseg/xseg_exports.h b/xseg/xseg/xseg_exports.h
index aafae25..51bde71 100644 (file)
--- a/xseg/xseg/xseg_exports.h
+++ b/xseg/xseg/xseg_exports.h
@@ -10,6 +10,7 @@ EXPORT_SYMBOL(xseg_disable_driver);
 EXPORT_SYMBOL(xseg_create);
 EXPORT_SYMBOL(xseg_destroy);
 EXPORT_SYMBOL(xseg_join);
+EXPORT_SYMBOL(xseg_leave);
 EXPORT_SYMBOL(xseg_bind_port);
 EXPORT_SYMBOL(xseg_alloc_requests);
 EXPORT_SYMBOL(xseg_free_requests);