Merge pull request #7 from jfckm/live-recovery

Live recovery

Author: Robert Baldyga

module/bdev/ocf/ctx.c

@@ -30,6 +30,12 @@
  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
+
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fcntl.h>
+
 #include <ocf/ocf.h>
 #include <execinfo.h>
 
@@ -590,6 +596,195 @@ vbdev_ocf_ctx_log_printf(ocf_logger_t logger, ocf_logger_lvl_t lvl,
 	return 0;
 }
 
+struct ocf_cache_persistent_meta_segment_descriptor {
+	int id;
+
+	bool valid;
+	size_t size;
+	size_t offset;
+};
+
+#define MAX_SEGMENTS 20
+struct shm_superblock {
+	struct ocf_cache_persistent_meta_segment_descriptor segments[MAX_SEGMENTS];
+};
+
+
+static ocf_persistent_meta_zone_t
+vbdev_ocf_persistent_meta_init(ocf_cache_t cache, size_t size, bool *load)
+{
+	struct vbdev_ocf_cache_ctx *ctx = ocf_cache_get_priv(cache);
+	struct ocf_persistent_meta_zone *pmeta;
+	struct stat stat_buf;
+	void *shm;
+	struct shm_superblock *shm_sb;
+	int zone;
+
+	size += sizeof(struct shm_superblock);
+
+	if (!ctx->create && ctx->force)
+		return NULL;
+
+	for (zone = 0; zone < MAX_PERSISTENT_ZONES; zone++) {
+		if (!ctx->persistent_meta[zone].fd)
+			break;
+	}
+
+	if (zone >= MAX_PERSISTENT_ZONES)
+		return NULL;
+
+
+	pmeta = &ctx->persistent_meta[zone];
+	*load = false;
+
+	snprintf(pmeta->name, NAME_MAX, "/ocf.%s.%u", ctx->cache_name, zone);
+
+	pmeta->fd = shm_open(pmeta->name, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
+	if (pmeta->fd < 0) {
+		SPDK_ERRLOG("Can't open SHM %s\n", pmeta->name);
+		return NULL;
+	}
+
+	if (ctx->force) {
+		SPDK_NOTICELOG("Using force, truncating SHM %s\n",
+				pmeta->name);
+		if (ftruncate(pmeta->fd, 0) < 0) {
+			SPDK_ERRLOG("Failed to truncate\n");
+			goto err_stat;
+		}
+
+	} else if (fstat(pmeta->fd, &stat_buf) < 0) {
+		SPDK_ERRLOG("Can't stat SHM %s\n", pmeta->name);
+		goto err_stat;
+	}
+
+	if (!ctx->force && stat_buf.st_size != 0) {
+		if (ctx->create) {
+			SPDK_ERRLOG("SHM %s found, but no force specified!\n",
+					pmeta->name);
+			goto err_stat;
+		}
+
+		*load = true;
+		SPDK_NOTICELOG("Loading from SHM\n");
+		if (stat_buf.st_size < size) {
+			if (ftruncate(pmeta->fd, size) < 0) {
+				SPDK_ERRLOG("Failed to extend SHM\n");
+				goto err_stat;
+			}
+			pmeta->size = size;
+		} else if (stat_buf.st_size == size) {
+			pmeta->size = size;
+		} else {
+			SPDK_NOTICELOG("Refusing to shrink SHM\n");
+			pmeta->size = stat_buf.st_size;
+		}
+	} else {
+		SPDK_NOTICELOG("Set SHM size\n");
+		if (ftruncate(pmeta->fd, size) < 0) {
+			SPDK_ERRLOG("Failed to truncate\n");
+			goto err_stat;
+		}
+
+		pmeta->size = size;
+	}
+
+	shm = mmap(NULL, pmeta->size, PROT_READ | PROT_WRITE, MAP_SHARED, pmeta->fd, 0);
+	if (shm == MAP_FAILED) {
+		SPDK_ERRLOG("Failed to map shm\n");
+		goto err_stat;
+	}
+
+	pmeta->data = shm;
+	shm_sb = shm;
+
+	if (mlock(shm, pmeta->size)) {
+		SPDK_ERRLOG("Failed to mlock\n");
+
+		goto err_mlock;
+	}
+
+
+	if (!*load) {
+		shm_sb->segments[0].size = sizeof(struct shm_superblock);
+		shm_sb->segments[0].offset = 0;
+		shm_sb->segments[0].id = -1;
+		shm_sb->segments[0].valid = true;
+	}
+
+	return pmeta;
+
+err_mlock:
+	munmap(pmeta->data, pmeta->size);
+err_stat:
+	close(pmeta->fd);
+	shm_unlink(pmeta->name);
+
+	return NULL;
+}
+
+static int
+vbdev_ocf_persistent_meta_deinit(struct ocf_persistent_meta_zone *zone)
+{
+	/*
+	munmap(zone->data, zone->size);
+	close(zone->fd);
+	shm_unlink(zone->name);
+	*/
+
+	return 0;
+}
+
+static void *
+vbdev_ocf_persistent_meta_alloc(struct ocf_persistent_meta_zone *zone, size_t size,
+		int alloc_id, bool *load)
+{
+	int i;
+	size_t sum = 0;
+	struct shm_superblock *shm_sb = zone->data;
+	struct ocf_cache_persistent_meta_segment_descriptor *desc =
+		shm_sb->segments;
+
+	alloc_id++;
+	*load = false;
+
+	for (i = 0; i < MAX_SEGMENTS; i++) {
+		/* assume no fragmentation */
+		if (!desc[i].valid)
+			break;
+
+		sum += desc[i].size;
+
+		/* assume no size change */
+		if (desc[i].id == alloc_id) {
+			*load = true;
+			return zone->data + desc[i].offset;
+		}
+	}
+
+	if (i == MAX_SEGMENTS)
+		return NULL;
+
+	 /* TODO: align */
+
+	if (sum + size > zone->size)
+		return NULL;
+
+	desc[i].id = alloc_id;
+	desc[i].offset = sum;
+	desc[i].size = size;
+	desc[i].valid = true;
+
+	return zone->data + desc[i].offset;
+}
+
+static int
+vbdev_ocf_persistent_meta_free(struct ocf_persistent_meta_zone *zone, int alloc_id,
+		void *ptr)
+{
+	return 0;
+}
+
 static const struct ocf_ctx_config vbdev_ocf_ctx_cfg = {
 	.name = "OCF SPDK",
 
@@ -613,6 +808,13 @@ static const struct ocf_ctx_config vbdev_ocf_ctx_cfg = {
 			.kick = vbdev_ocf_volume_updater_kick,
 		},
 
+		.persistent_meta = {
+			.init = vbdev_ocf_persistent_meta_init,
+			.deinit = vbdev_ocf_persistent_meta_deinit,
+			.alloc = vbdev_ocf_persistent_meta_alloc,
+			.free = vbdev_ocf_persistent_meta_free,
+		},
+
 		.cleaner = {
 			.init = vbdev_ocf_ctx_cleaner_init,
 			.stop = vbdev_ocf_ctx_cleaner_stop,

module/bdev/ocf/ctx.h

@@ -44,6 +44,15 @@ extern ocf_ctx_t vbdev_ocf_ctx;
 
 #define SPDK_OBJECT 1
 
+struct ocf_persistent_meta_zone {
+	int fd;
+	size_t size;
+	char name[NAME_MAX];
+	void *data;
+};
+
+#define MAX_PERSISTENT_ZONES 2
+
 /* Context of cache instance */
 struct vbdev_ocf_cache_ctx {
 	struct vbdev_ocf             *vbdev;
@@ -55,6 +64,10 @@ struct vbdev_ocf_cache_ctx {
 	/* Channels for cleaner */
 	struct spdk_io_channel      *cleaner_cache_channel;
 	struct spdk_io_channel      *cleaner_core_channel;
+	bool                         create;
+	bool                         force;
+	char cache_name[OCF_CACHE_NAME_SIZE];
+	struct ocf_persistent_meta_zone    persistent_meta[MAX_PERSISTENT_ZONES];
 };
 
 void vbdev_ocf_cache_ctx_put(struct vbdev_ocf_cache_ctx *ctx);

module/bdev/ocf/vbdev_ocf.c

@@ -746,12 +746,12 @@ vbdev_ocf_dump_info_json(void *opaque, struct spdk_json_write_ctx *w)
 				     ocf_get_cache_modename(ocf_cache_get_mode(vbdev->ocf_cache)));
 	spdk_json_write_named_uint32(w, "cache_line_size",
 				     ocf_get_cache_line_size(vbdev->ocf_cache));
-	spdk_json_write_named_bool(w, "metadata_volatile",
-				   vbdev->cfg.cache.metadata_volatile);
 	if (vbdev->cpu_mask)
 	{
 		spdk_json_write_named_string(w, "cpu_mask", vbdev->cpu_mask);
 	}
+	spdk_json_write_named_uint32(w, "metadata_persistence_mode",
+				   vbdev->cfg.cache.persistence_mode);
 
 	return 0;
 }
@@ -1170,11 +1170,18 @@ start_cache(struct vbdev_ocf *vbdev)
 		return;
 	}
 
+	vbdev->cache_ctx->create = !vbdev->cfg.loadq;
+	vbdev->cache_ctx->force = vbdev->cfg.device.force;
+
 	vbdev_ocf_cache_ctx_get(vbdev->cache_ctx);
 	vbdev->cache_ctx->vbdev = vbdev;
 	pthread_mutex_init(&vbdev->cache_ctx->lock, NULL);
 
-	rc = ocf_mngt_cache_start(vbdev_ocf_ctx, &vbdev->ocf_cache, &vbdev->cfg.cache, vbdev->cache_ctx);
+	env_strncpy(vbdev->cache_ctx->cache_name, OCF_CACHE_NAME_SIZE,
+			vbdev->cfg.cache.name, OCF_CACHE_NAME_SIZE);
+
+	rc = ocf_mngt_cache_start(vbdev_ocf_ctx, &vbdev->ocf_cache,
+			&vbdev->cfg.cache, vbdev->cache_ctx);
 	if (rc) {
 		SPDK_ERRLOG("Could not start cache %s: %d\n", vbdev->name, rc);
 		vbdev_ocf_mngt_exit(vbdev, unregister_path_dirty, rc);
@@ -1237,7 +1244,7 @@ init_vbdev_config(struct vbdev_ocf *vbdev)
 
 	/* TODO [metadata]: make configurable with persistent
 	 * metadata support */
-	cfg->cache.metadata_volatile = false;
+	cfg->cache.persistence_mode = ocf_metadata_persistence_persistent;
 
 	/* This are suggested values that
 	 * should be sufficient for most use cases */