diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig index 7b2df7a54d87..0dd0cda6e543 100644 --- a/drivers/block/Kconfig +++ b/drivers/block/Kconfig @@ -485,3 +485,23 @@ config BLK_DEV_RSXX module will be called rsxx. endif # BLK_DEV + +config VSERVICES_BLOCK_SERVER + tristate "Virtual Services block server" + depends on BLOCK && VSERVICES_SUPPORT && VSERVICES_SERVER + default y + select VSERVICES_PROTOCOL_BLOCK_SERVER + help + Select this option if you want support for server side Virtual + Services block. This allows any Linux block device to be + virtualized and exported as a virtual service. + +config VSERVICES_BLOCK_CLIENT + tristate "Virtual Services Block client device" + depends on BLOCK && VSERVICES_SUPPORT && VSERVICES_CLIENT + default y + select VSERVICES_PROTOCOL_BLOCK_CLIENT + help + Select this option if you want support for client side Virtual + Services block devices. The virtual block devices are typically + named /dev/vblock0, /dev/vblock1, etc. diff --git a/drivers/block/Makefile b/drivers/block/Makefile index dc061158b403..9f1d4637ad0f 100644 --- a/drivers/block/Makefile +++ b/drivers/block/Makefile @@ -41,3 +41,8 @@ obj-$(CONFIG_ZRAM) += zram/ skd-y := skd_main.o swim_mod-y := swim.o swim_asm.o + +obj-$(CONFIG_VSERVICES_BLOCK_SERVER) += vs_block_server.o +CFLAGS_vs_block_server.o += -Werror +obj-$(CONFIG_VSERVICES_BLOCK_CLIENT) += vs_block_client.o +CFLAGS_vs_block_client.o += -Werror diff --git a/drivers/block/vs_block_client.c b/drivers/block/vs_block_client.c new file mode 100644 index 000000000000..974f8b9874aa --- /dev/null +++ b/drivers/block/vs_block_client.c @@ -0,0 +1,956 @@ +/* + * drivers/block/vs_block_client.c + * + * Copyright (c) 2012-2018 General Dynamics + * Copyright (c) 2014 Open Kernel Labs, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * block vservice client driver + * + * Function vs_block_client_vs_alloc() is partially derived from + * drivers/block/brd.c (brd_alloc()) + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +/* + * BLK_DEF_MAX_SECTORS was replaced with the hard-coded number 1024 in 3.19, + * and restored in 4.3 + */ +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) && \ + (LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0)) +#define BLK_DEF_MAX_SECTORS 1024 +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0) +#define bio_sector(bio) (bio)->bi_iter.bi_sector +#define bio_size(bio) (bio)->bi_iter.bi_size +#else +#define bio_sector(bio) (bio)->bi_sector +#define bio_size(bio) (bio)->bi_size +#endif + +#define CLIENT_BLKDEV_NAME "vblock" + +#define PERDEV_MINORS 256 + +struct block_client; + +struct vs_block_device { + /* + * The client that created this block device. A reference is held + * to the client until the block device is released, so this pointer + * should always be valid. However, the client may since have reset; + * so it should only be used if, after locking it, its blkdev pointer + * points back to this block device. + */ + struct block_client *client; + + int id; + struct gendisk *disk; + struct request_queue *queue; + + struct kref kref; +}; + +struct block_client { + struct vs_client_block_state client; + struct vs_service_device *service; + + /* Tasklet & queue for bouncing buffers out of read acks */ + struct tasklet_struct rx_tasklet; + struct list_head rx_queue; + struct spinlock rx_queue_lock; + + /* + * The current virtual block device. This gets replaced when we do + * a reset since other parts of the kernel (e.g. vfs) may still + * be accessing the disk. + */ + struct vs_block_device *blkdev; + + /* Shared work item for disk creation */ + struct work_struct disk_creation_work; + + struct kref kref; +}; + +#define state_to_block_client(state) \ + container_of(state, struct block_client, client) + +static int block_client_major; + +/* Unique identifier allocation for virtual block devices */ +static DEFINE_IDA(vs_block_ida); +static DEFINE_MUTEX(vs_block_ida_lock); + +static int +block_client_vs_to_linux_error(vservice_block_block_io_error_t vs_err) +{ + switch (vs_err) { + case VSERVICE_BLOCK_INVALID_INDEX: + return -EILSEQ; + case VSERVICE_BLOCK_MEDIA_FAILURE: + return -EIO; + case VSERVICE_BLOCK_MEDIA_TIMEOUT: + return -ETIMEDOUT; + case VSERVICE_BLOCK_UNSUPPORTED_COMMAND: + return -ENOTSUPP; + case VSERVICE_BLOCK_SERVICE_RESET: + return -ENXIO; + default: + WARN_ON(vs_err); + return 0; + } + + return 0; +} + +static void vs_block_client_kfree(struct kref *kref) +{ + struct block_client *client = + container_of(kref, struct block_client, kref); + + vs_put_service(client->service); + kfree(client); +} + +static void vs_block_client_put(struct block_client *client) +{ + kref_put(&client->kref, vs_block_client_kfree); +} + +static void vs_block_device_kfree(struct kref *kref) +{ + struct vs_block_device *blkdev = + container_of(kref, struct vs_block_device, kref); + + /* Delete the disk and clean up its queue */ + del_gendisk(blkdev->disk); + blk_cleanup_queue(blkdev->queue); + put_disk(blkdev->disk); + + mutex_lock(&vs_block_ida_lock); + ida_remove(&vs_block_ida, blkdev->id); + mutex_unlock(&vs_block_ida_lock); + + if (blkdev->client) + vs_block_client_put(blkdev->client); + + kfree(blkdev); +} + +static void vs_block_device_put(struct vs_block_device *blkdev) +{ + kref_put(&blkdev->kref, vs_block_device_kfree); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0) +static void +#else +static int +#endif +vs_block_client_blkdev_release(struct gendisk *disk, fmode_t mode) +{ + struct vs_block_device *blkdev = disk->private_data; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0) + if (WARN_ON(!blkdev)) + return; +#else + if (WARN_ON(!blkdev)) + return -ENXIO; +#endif + + vs_block_device_put(blkdev); +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0) + return 0; +#endif +} + +static int vs_block_client_blkdev_open(struct block_device *bdev, fmode_t mode) +{ + struct vs_block_device *blkdev = bdev->bd_disk->private_data; + struct block_client *client; + int err = -ENXIO; + + if (!blkdev || !kref_get_unless_zero(&blkdev->kref)) + goto fail_get_blkdev; + + client = blkdev->client; + if (WARN_ON(!client)) + goto fail_lock_client; + + if (!vs_state_lock_safe(&client->client)) { + err = -ENODEV; + goto fail_lock_client; + } + + if (blkdev != client->blkdev) { + /* The client has reset, this blkdev is no longer usable */ + err = -ENXIO; + goto fail_check_client; + } + + if ((mode & FMODE_WRITE) > 0 && client->client.readonly) { + dev_dbg(&client->service->dev, + "opening a readonly disk as writable\n"); + err = -EROFS; + goto fail_check_client; + } + + vs_state_unlock(&client->client); + + return 0; + +fail_check_client: + vs_state_unlock(&client->client); +fail_lock_client: + vs_block_device_put(blkdev); +fail_get_blkdev: + return err; +} + +static int vs_block_client_blkdev_getgeo(struct block_device *bdev, + struct hd_geometry *geo) +{ + /* These numbers are some default sane values for disk geometry. */ + geo->cylinders = get_capacity(bdev->bd_disk) / (4 * 16); + geo->heads = 4; + geo->sectors = 16; + + return 0; +} + +/* + * Indirectly determine linux block layer sector size and ensure that our + * sector size matches. + */ +static int vs_block_client_check_sector_size(struct block_client *client, + struct bio *bio) +{ + unsigned int expected_bytes; + + if (unlikely(!bio_sectors(bio))) { + dev_err(&client->service->dev, "zero-length bio"); + return -EIO; + } + + expected_bytes = bio_sectors(bio) * client->client.sector_size; + if (unlikely(bio_size(bio) != expected_bytes)) { + dev_err(&client->service->dev, + "bio has %zd bytes, which is unexpected " + "for %d sectors of %zd bytes each", + (size_t)bio_size(bio), bio_sectors(bio), + (size_t)client->client.sector_size); + return -EIO; + } + + return 0; +} + +static const struct block_device_operations block_client_ops = { + .getgeo = vs_block_client_blkdev_getgeo, + .open = vs_block_client_blkdev_open, + .release = vs_block_client_blkdev_release, + .owner = THIS_MODULE, +}; + +static int block_client_send_write_req(struct block_client *client, + struct bio *bio) +{ + struct vs_client_block_state *state = &client->client; + struct vs_mbuf *mbuf; + struct vs_pbuf pbuf; + struct bio_vec *bvec; + int err; + bool flush, nodelay, commit; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0) + struct bvec_iter iter; + struct bio_vec bvec_local; +#else + int i; +#endif + + err = vs_block_client_check_sector_size(client, bio); + if (err < 0) + goto fail; + + do { + /* Wait until it's possible to send a write request */ + err = vs_wait_state_nointr(state, + vs_client_block_io_req_write_can_send(state)); + if (err == -ECANCELED) + err = -ENXIO; + if (err < 0) + goto fail; + + /* Wait for quota, while sending a write remains possible */ + mbuf = vs_wait_alloc_nointr(state, + vs_client_block_io_req_write_can_send(state), + vs_client_block_io_alloc_req_write( + state, &pbuf, GFP_KERNEL)); + err = IS_ERR(mbuf) ? PTR_ERR(mbuf) : 0; + + /* Retry if sending is no longer possible */ + } while (err == -ECANCELED); + + if (err < 0) + goto fail; + + vs_pbuf_resize(&pbuf, 0); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0) + bvec = &bvec_local; + bio_for_each_segment(bvec_local, bio, iter) +#else + bio_for_each_segment(bvec, bio, i) +#endif + { + unsigned long flags; + void *buf = bvec_kmap_irq(bvec, &flags); + flush_kernel_dcache_page(bvec->bv_page); + err = vs_pbuf_append(&pbuf, buf, bvec->bv_len); + bvec_kunmap_irq(buf, &flags); + if (err < 0) { + dev_err(&client->service->dev, + "pbuf copy failed with err %d\n", err); + err = -EIO; + goto fail_free_write; + } + } + + if (unlikely(vs_pbuf_size(&pbuf) != bio_size(bio))) { + dev_err(&client->service->dev, + "pbuf size is wrong: %zd, should be %zd\n", + vs_pbuf_size(&pbuf), (size_t)bio_size(bio)); + err = -EIO; + goto fail_free_write; + } +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,8,0) + flush = (bio_flags(bio) & REQ_PREFLUSH); + commit = (bio_flags(bio) & REQ_FUA); + nodelay = (bio_flags(bio) & REQ_SYNC); +#else + flush = (bio->bi_rw & REQ_FLUSH); + commit = (bio->bi_rw & REQ_FUA); + nodelay = (bio->bi_rw & REQ_SYNC); +#endif + err = vs_client_block_io_req_write(state, bio, bio_sector(bio), + bio_sectors(bio), nodelay, flush, commit, pbuf, mbuf); + + if (err) { + dev_err(&client->service->dev, + "write req failed with err %d\n", err); + goto fail_free_write; + } + + return 0; + +fail_free_write: + vs_client_block_io_free_req_write(state, &pbuf, mbuf); +fail: + return err; +} + +static int block_client_send_read_req(struct block_client *client, + struct bio *bio) +{ + struct vs_client_block_state *state = &client->client; + int err; + bool flush, nodelay; + + err = vs_block_client_check_sector_size(client, bio); + if (err < 0) + return err; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,8,0) + flush = (bio_flags(bio) & REQ_PREFLUSH); + nodelay = (bio_flags(bio) & REQ_SYNC); +#else + flush = (bio->bi_rw & REQ_FLUSH); + nodelay = (bio->bi_rw & REQ_SYNC); +#endif + do { + /* Wait until it's possible to send a read request */ + err = vs_wait_state_nointr(state, + vs_client_block_io_req_read_can_send(state)); + if (err == -ECANCELED) + err = -ENXIO; + if (err < 0) + break; + + /* Wait for quota, while sending a read remains possible */ + err = vs_wait_send_nointr(state, + vs_client_block_io_req_read_can_send(state), + vs_client_block_io_req_read(state, bio, + bio_sector(bio), bio_sectors(bio), + nodelay, flush, GFP_KERNEL)); + } while (err == -ECANCELED); + + return err; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +static blk_qc_t +#else +static void +#endif +vs_block_client_make_request(struct request_queue *q, struct bio *bio) +{ + struct block_device *bdev = bio->bi_bdev; + struct vs_block_device *blkdev = bdev->bd_disk->private_data; + struct block_client *client; + int err = 0; + + client = blkdev->client; + if (!client || !kref_get_unless_zero(&client->kref)) { + err = -ENODEV; + goto fail_get_client; + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0) + blk_queue_split(q, &bio, q->bio_split); +#endif + + if (!vs_state_lock_safe(&client->client)) { + err = -ENODEV; + goto fail_lock_client; + } + + if (client->blkdev != blkdev) { + /* Client has reset, this block device is no longer usable */ + err = -EIO; + goto fail_check_client; + } + + if (bio_data_dir(bio) == WRITE) + err = block_client_send_write_req(client, bio); + else + err = block_client_send_read_req(client, bio); + +fail_check_client: + if (err == -ENOLINK) + err = -EIO; + else + vs_state_unlock(&client->client); +fail_lock_client: + vs_block_client_put(client); +fail_get_client: +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0) + if (err < 0) { + bio->bi_error = err; + bio_endio(bio); + } +#else + if (err < 0) + bio_endio(bio, err); +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) + return BLK_QC_T_NONE; +#endif +} + +static int vs_block_client_get_blkdev_id(struct block_client *client) +{ + int id; + int ret; + +retry: + ret = ida_pre_get(&vs_block_ida, GFP_KERNEL); + if (ret == 0) + return -ENOMEM; + + mutex_lock(&vs_block_ida_lock); + ret = ida_get_new(&vs_block_ida, &id); + mutex_unlock(&vs_block_ida_lock); + + if (ret == -EAGAIN) + goto retry; + + return id; +} + +static int vs_block_client_disk_add(struct block_client *client) +{ + struct vs_block_device *blkdev; + unsigned int max_hw_sectors; + int err; + + dev_dbg(&client->service->dev, "device add\n"); + + blkdev = kzalloc(sizeof(*blkdev), GFP_KERNEL); + if (!blkdev) { + err = -ENOMEM; + goto fail; + } + + kref_init(&blkdev->kref); + blkdev->id = vs_block_client_get_blkdev_id(client); + if (blkdev->id < 0) { + err = blkdev->id; + goto fail_free_blkdev; + } + + if ((blkdev->id * PERDEV_MINORS) >> MINORBITS) { + err = -ENODEV; + goto fail_remove_ida; + } + + blkdev->queue = blk_alloc_queue(GFP_KERNEL); + if (!blkdev->queue) { + dev_err(&client->service->dev, + "Error initializing blk queue\n"); + err = -ENOMEM; + goto fail_remove_ida; + } + + blk_queue_make_request(blkdev->queue, vs_block_client_make_request); + blk_queue_bounce_limit(blkdev->queue, BLK_BOUNCE_ANY); + blk_queue_dma_alignment(blkdev->queue, 0); + + /* + * Mark this as a paravirtualised device. This is just an alias + * of QUEUE_FLAG_NONROT, which prevents the I/O schedulers trying + * to wait for the disk to spin. + */ + queue_flag_set_unlocked(QUEUE_FLAG_VIRT, blkdev->queue); + + blkdev->queue->queuedata = blkdev; + + blkdev->client = client; + kref_get(&client->kref); + + max_hw_sectors = min_t(sector_t, BLK_DEF_MAX_SECTORS, + client->client.segment_size / + client->client.sector_size); + blk_queue_max_hw_sectors(blkdev->queue, max_hw_sectors); + + blkdev->disk = alloc_disk(PERDEV_MINORS); + if (!blkdev->disk) { + dev_err(&client->service->dev, "Error allocating disk\n"); + err = -ENOMEM; + goto fail_free_blk_queue; + } + + if (client->client.readonly) { + dev_dbg(&client->service->dev, "set device as readonly\n"); + set_disk_ro(blkdev->disk, true); + } + + blkdev->disk->major = block_client_major; + blkdev->disk->first_minor = blkdev->id * PERDEV_MINORS; + blkdev->disk->fops = &block_client_ops; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,7,0) + blkdev->disk->driverfs_dev = &client->service->dev; +#endif + blkdev->disk->private_data = blkdev; + blkdev->disk->queue = blkdev->queue; + blkdev->disk->flags |= GENHD_FL_EXT_DEVT; + + /* + * The block device name is vblock, where x is a unique + * identifier. Userspace should rename or symlink the device + * appropriately, typically by processing the add uevent. + * + * If a virtual block device is reset then it may re-open with a + * different identifier if something still holds a reference to + * the old device (such as a userspace application having an open + * file handle). + */ + snprintf(blkdev->disk->disk_name, sizeof(blkdev->disk->disk_name), + "%s%d", CLIENT_BLKDEV_NAME, blkdev->id); + set_capacity(blkdev->disk, client->client.device_sectors); + + /* + * We need to hold a reference on blkdev across add_disk(), to make + * sure a concurrent reset does not immediately release the blkdev + * and call del_gendisk(). + */ + kref_get(&blkdev->kref); + + vs_service_state_lock(client->service); + if (!VSERVICE_BASE_STATE_IS_RUNNING(client->client.state.base)) { + vs_service_state_unlock(client->service); + err = -ENXIO; + goto fail_free_blk_queue; + } + client->blkdev = blkdev; + vs_service_state_unlock(client->service); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,7,0) + device_add_disk(&client->service->dev, blkdev->disk); +#else + add_disk(blkdev->disk); +#endif + dev_dbg(&client->service->dev, "added block disk '%s'\n", + blkdev->disk->disk_name); + + /* Release the reference taken above. */ + vs_block_device_put(blkdev); + + return 0; + +fail_free_blk_queue: + blk_cleanup_queue(blkdev->queue); +fail_remove_ida: + mutex_lock(&vs_block_ida_lock); + ida_remove(&vs_block_ida, blkdev->id); + mutex_unlock(&vs_block_ida_lock); +fail_free_blkdev: + kfree(blkdev); +fail: + return err; +} + +static void vs_block_client_disk_creation_work(struct work_struct *work) +{ + struct block_client *client = container_of(work, + struct block_client, disk_creation_work); + struct vs_block_device *blkdev; + bool running; + + vs_service_state_lock(client->service); + blkdev = client->blkdev; + running = VSERVICE_BASE_STATE_IS_RUNNING(client->client.state.base); + + dev_dbg(&client->service->dev, + "disk changed: blkdev = %pK, running = %d\n", + client->blkdev, running); + if (!blkdev && running) { + dev_dbg(&client->service->dev, "adding block disk\n"); + vs_service_state_unlock(client->service); + vs_block_client_disk_add(client); + } else { + vs_service_state_unlock(client->service); + } +} + +static void vs_block_client_rx_tasklet(unsigned long data); + +static struct vs_client_block_state * +vs_block_client_alloc(struct vs_service_device *service) +{ + struct block_client *client; + + client = kzalloc(sizeof(*client), GFP_KERNEL); + if (!client) { + dev_err(&service->dev, "Error allocating client struct\n"); + return NULL; + } + + vs_get_service(service); + client->service = service; + + INIT_LIST_HEAD(&client->rx_queue); + spin_lock_init(&client->rx_queue_lock); + tasklet_init(&client->rx_tasklet, vs_block_client_rx_tasklet, + (unsigned long)client); + tasklet_disable(&client->rx_tasklet); + + INIT_WORK(&client->disk_creation_work, + vs_block_client_disk_creation_work); + kref_init(&client->kref); + + dev_dbg(&service->dev, "New block client %pK\n", client); + + return &client->client; +} + +static void vs_block_client_release(struct vs_client_block_state *state) +{ + struct block_client *client = state_to_block_client(state); + + flush_work(&client->disk_creation_work); + + vs_block_client_put(client); +} + +/* FIXME: Jira ticket SDK-2459 - anjaniv */ +static void vs_block_client_closed(struct vs_client_block_state *state) +{ + struct block_client *client = state_to_block_client(state); + + /* + * Stop the RX bounce tasklet and clean up its queue. We can wait for + * it to stop safely because it doesn't need to acquire the state + * lock, only the RX lock which we acquire after it is disabled. + */ + tasklet_disable(&client->rx_tasklet); + spin_lock(&client->rx_queue_lock); + while (!list_empty(&client->rx_queue)) { + struct vs_mbuf *mbuf = list_first_entry(&client->rx_queue, + struct vs_mbuf, queue); + struct vs_pbuf pbuf; + list_del(&mbuf->queue); + vs_client_block_io_getbufs_ack_read(state, &pbuf, mbuf); + vs_client_block_io_free_ack_read(state, &pbuf, mbuf); + } + spin_unlock(&client->rx_queue_lock); + + if (client->blkdev) { + struct vs_block_device *blkdev = client->blkdev; + char service_remove[] = "REMOVING_SERVICE=1"; + /* + 9 because "DEVNAME=" is 8 chars plus 1 for '\0' */ + char devname[sizeof(blkdev->disk->disk_name) + 9]; + char *envp[] = { service_remove, devname, NULL }; + + dev_dbg(&client->service->dev, "removing block disk\n"); + + /* + * Send a change event with DEVNAME to allow the block helper + * script to remove any server sessions which use either + * v${SERVICE_NAME} or ${DEVNAME}. The remove event generated + * by the session driver doesn't include DEVNAME so the only + * way for userspace to map SERVICE_NAME to DEVNAME is by the + * symlink added when the client service was created. If that + * symlink has been deleted, there's no other way to connect + * the two names. + */ + snprintf(devname, sizeof(devname), "DEVNAME=%s", + blkdev->disk->disk_name); + kobject_uevent_env(&client->service->dev.kobj, KOBJ_CHANGE, + envp); + + /* + * We are done with the device now. The block device will only + * get removed once there are no more users (e.g. userspace + * applications). + */ + client->blkdev = NULL; + vs_block_device_put(blkdev); + } +} + +static void vs_block_client_opened(struct vs_client_block_state *state) +{ + struct block_client *client = state_to_block_client(state); + +#if !defined(CONFIG_LBDAF) && !defined(CONFIG_64BIT) + if (state->device_sectors >> (sizeof(sector_t) * 8)) { + dev_err(&client->service->dev, + "Client doesn't support full capacity large block devices\n"); + vs_client_block_close(state); + return; + } +#endif + + /* Unblock the RX bounce tasklet. */ + tasklet_enable(&client->rx_tasklet); + + /* + * The block device allocation needs to sleep, so we defer it to a + * work queue. + */ + queue_work(client->service->work_queue, &client->disk_creation_work); +} + +static int vs_block_client_ack_read(struct vs_client_block_state *state, + void *tag, struct vs_pbuf pbuf, struct vs_mbuf *mbuf) +{ + struct block_client *client = state_to_block_client(state); + struct bio *bio = tag; + struct bio_vec *bvec; + int err = 0; + size_t bytes_read = 0; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0) + struct bio_vec bvec_local; + struct bvec_iter iter; +#else + int i; +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0) + bvec = &bvec_local; + bio_for_each_segment(bvec_local, bio, iter) +#else + bio_for_each_segment(bvec, bio, i) +#endif + { + unsigned long flags; + void *buf; + if (vs_pbuf_size(&pbuf) < bytes_read + bvec->bv_len) { + dev_err(&client->service->dev, + "bio read overrun: %zu into %zu byte response, but need %zd bytes\n", + bytes_read, vs_pbuf_size(&pbuf), + (size_t)bvec->bv_len); + err = -EIO; + break; + } + buf = bvec_kmap_irq(bvec, &flags); + memcpy(buf, vs_pbuf_data(&pbuf) + bytes_read, bvec->bv_len); + flush_kernel_dcache_page(bvec->bv_page); + bvec_kunmap_irq(buf, &flags); + bytes_read += bvec->bv_len; + } + + vs_client_block_io_free_ack_read(state, &pbuf, mbuf); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0) + if (err < 0) + bio->bi_error = err; + bio_endio(bio); +#else + bio_endio(bio, err); +#endif + + return 0; +} + +static void vs_block_client_rx_tasklet(unsigned long data) +{ + struct block_client *client = (struct block_client *)data; + struct vs_mbuf *mbuf; + struct vs_pbuf pbuf; + + spin_lock(&client->rx_queue_lock); + + /* The list shouldn't be empty. */ + if (WARN_ON(list_empty(&client->rx_queue))) { + spin_unlock(&client->rx_queue_lock); + return; + } + + /* Get the next mbuf, and reschedule ourselves if there are more. */ + mbuf = list_first_entry(&client->rx_queue, struct vs_mbuf, queue); + list_del(&mbuf->queue); + if (!list_empty(&client->rx_queue)) + tasklet_schedule(&client->rx_tasklet); + + spin_unlock(&client->rx_queue_lock); + + /* Process the ack. */ + vs_client_block_io_getbufs_ack_read(&client->client, &pbuf, mbuf); + vs_block_client_ack_read(&client->client, mbuf->priv, pbuf, mbuf); +} + +static int vs_block_client_queue_ack_read(struct vs_client_block_state *state, + void *tag, struct vs_pbuf pbuf, struct vs_mbuf *mbuf) +{ + struct block_client *client = state_to_block_client(state); + + spin_lock(&client->rx_queue_lock); + list_add_tail(&mbuf->queue, &client->rx_queue); + mbuf->priv = tag; + spin_unlock(&client->rx_queue_lock); + + tasklet_schedule(&client->rx_tasklet); + + wake_up(&state->service->quota_wq); + + return 0; +} + +static int vs_block_client_ack_write(struct vs_client_block_state *state, + void *tag) +{ + struct bio *bio = tag; + + if (WARN_ON(!bio)) + return -EPROTO; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0) + bio_endio(bio); +#else + bio_endio(bio, 0); +#endif + + wake_up(&state->service->quota_wq); + + return 0; +} + +static int vs_block_client_nack_io(struct vs_client_block_state *state, + void *tag, vservice_block_block_io_error_t err) +{ + struct bio *bio = tag; + + if (WARN_ON(!bio)) + return -EPROTO; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0) + bio->bi_error = block_client_vs_to_linux_error(err); + bio_endio(bio); +#else + bio_endio(bio, block_client_vs_to_linux_error(err)); +#endif + + wake_up(&state->service->quota_wq); + + return 0; +} + +static struct vs_client_block block_client_driver = { + .rx_atomic = true, + .alloc = vs_block_client_alloc, + .release = vs_block_client_release, + .opened = vs_block_client_opened, + .closed = vs_block_client_closed, + .io = { + .ack_read = vs_block_client_queue_ack_read, + .nack_read = vs_block_client_nack_io, + .ack_write = vs_block_client_ack_write, + .nack_write = vs_block_client_nack_io, + } +}; + +static int __init vs_block_client_init(void) +{ + int err; + + block_client_major = register_blkdev(0, CLIENT_BLKDEV_NAME); + if (block_client_major < 0) { + pr_err("Err registering blkdev\n"); + err = -ENOMEM; + goto fail; + } + + err = vservice_block_client_register(&block_client_driver, + "block_client_driver"); + if (err) + goto fail_unregister_blkdev; + + return 0; + +fail_unregister_blkdev: + unregister_blkdev(block_client_major, CLIENT_BLKDEV_NAME); +fail: + return err; +} + +static void __exit vs_block_client_exit(void) +{ + vservice_block_client_unregister(&block_client_driver); + unregister_blkdev(block_client_major, CLIENT_BLKDEV_NAME); +} + +module_init(vs_block_client_init); +module_exit(vs_block_client_exit); + +MODULE_DESCRIPTION("OKL4 Virtual Services Block Client Driver"); +MODULE_AUTHOR("Open Kernel Labs, Inc"); diff --git a/drivers/block/vs_block_server.c b/drivers/block/vs_block_server.c new file mode 100644 index 000000000000..9d20f6af4dc8 --- /dev/null +++ b/drivers/block/vs_block_server.c @@ -0,0 +1,1179 @@ +/* + * drivers/block/vs_block_server.c + * + * Copyright (c) 2012-2018 General Dynamics + * Copyright (c) 2014 Open Kernel Labs, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * block vservice server driver + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#define VS_BLOCK_BLKDEV_DEFAULT_MODE FMODE_READ +#define VS_BLOCK_BLK_DEF_SECTOR_SIZE 512 + +/* + * Metadata for a request. Note that the bio must be embedded at the end of + * this structure, because it is allocated from a bioset. + */ +struct block_server_request { + struct block_server *server; + u32 tagid; + u32 size; + int op_err; + struct list_head list; + struct vs_pbuf pbuf; + struct vs_mbuf *mbuf; + bool bounced; + bool submitted; + + struct bio bio; +}; + +struct block_server { + struct vs_server_block_state server; + struct vs_service_device *service; + + struct block_device *bdev; + struct bio_set *bioset; + + unsigned int sector_size; + bool started; + + /* Bounced writes are deferred to keep memcpy off service queue */ + struct list_head bounce_req_queue; + struct work_struct bounce_req_work; + spinlock_t bounce_req_lock; + + /* Count of outstanding requests submitted to block layer */ + atomic_t submitted_req_count; + wait_queue_head_t submitted_req_wq; + + /* Completions are deferred because end_io may be in atomic context */ + struct list_head completed_req_queue; + struct work_struct completed_req_work; + spinlock_t completed_req_lock; +}; + +#define state_to_block_server(state) \ + container_of(state, struct block_server, server) + +#define dev_to_block_server(dev) \ + state_to_block_server(dev_get_drvdata(dev)) + +static inline vservice_block_block_io_error_t +block_server_linux_to_vs_error(int err) +{ + /* + * This list is not exhaustive. For all other errors, we return + * unsupported_command. + */ + switch (err) { + case -ECOMM: + case -EIO: + case -ENOMEM: + return VSERVICE_BLOCK_MEDIA_FAILURE; + case -ETIME: + case -ETIMEDOUT: + return VSERVICE_BLOCK_MEDIA_TIMEOUT; + case -EILSEQ: + return VSERVICE_BLOCK_INVALID_INDEX; + default: + if (err) + return VSERVICE_BLOCK_UNSUPPORTED_COMMAND; + return 0; + } + + return 0; +} + +static inline u32 vs_req_num_sectors(struct block_server *server, + struct block_server_request *req) +{ + return req->size / server->sector_size; +} + +static inline u64 vs_req_sector_index(struct block_server_request *req) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0) + return req->bio.bi_iter.bi_sector; +#else + return req->bio.bi_sector; +#endif +} + +static void vs_block_server_closed(struct vs_server_block_state *state) +{ + struct block_server *server = state_to_block_server(state); + struct block_server_request *req; + + /* + * Fail all requests that haven't been sent to the block layer yet. + */ + spin_lock(&server->bounce_req_lock); + while (!list_empty(&server->bounce_req_queue)) { + req = list_first_entry(&server->bounce_req_queue, + struct block_server_request, list); + list_del(&req->list); + spin_unlock(&server->bounce_req_lock); + bio_io_error(&req->bio); + spin_lock(&server->bounce_req_lock); + } + spin_unlock(&server->bounce_req_lock); + + /* + * Wait until all outstanding requests to the block layer are + * complete. + */ + wait_event(server->submitted_req_wq, + !atomic_read(&server->submitted_req_count)); + + /* + * Discard all the completed requests. + */ + spin_lock_irq(&server->completed_req_lock); + while (!list_empty(&server->completed_req_queue)) { + req = list_first_entry(&server->completed_req_queue, + struct block_server_request, list); + list_del(&req->list); + if (req->mbuf) { + spin_unlock_irq(&server->completed_req_lock); + if (bio_data_dir(&req->bio) == WRITE) + vs_server_block_io_free_req_write(state, + &req->pbuf, req->mbuf); + else + vs_server_block_io_free_ack_read(state, + &req->pbuf, req->mbuf); + spin_lock_irq(&server->completed_req_lock); + } + bio_put(&req->bio); + } + spin_unlock_irq(&server->completed_req_lock); +} + +static ssize_t +vs_block_server_readonly_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct block_server *server = dev_to_block_server(dev); + int err; + unsigned long val; + + vs_service_state_lock(server->service); + if (server->started) { + err = -EBUSY; + goto unlock; + } + + err = kstrtoul(buf, 0, &val); + if (err) + goto unlock; + + if (bdev_read_only(server->bdev) && !val) { + dev_info(dev, + "Cannot set %s to read/write: read-only device\n", + server->service->name); + err = -EINVAL; + goto unlock; + } + + server->server.readonly = val; + err = count; + +unlock: + vs_service_state_unlock(server->service); + + return err; +} + +static ssize_t +vs_block_server_readonly_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct block_server *server = dev_to_block_server(dev); + int cnt; + + vs_service_state_lock(server->service); + cnt = scnprintf(buf, PAGE_SIZE, "%d\n", server->server.readonly); + vs_service_state_unlock(server->service); + + return cnt; +} + +static ssize_t +vs_block_server_start_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct block_server *server = dev_to_block_server(dev); + int err; + unsigned long val; + + vs_service_state_lock(server->service); + + err = kstrtoul(buf, 0, &val); + if (err) + goto unlock; + + if (!val && server->started) { + err = -EBUSY; + goto unlock; + } + + if (val && !server->started) { + server->started = true; + + if (server->server.state.base.statenum == + VSERVICE_BASE_STATE_CLOSED__OPEN) + vs_server_block_open_complete(&server->server, + VS_SERVER_RESP_SUCCESS); + } + + err = count; +unlock: + vs_service_state_unlock(server->service); + + return err; +} + +static ssize_t +vs_block_server_start_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct block_server *server = dev_to_block_server(dev); + int cnt; + + vs_service_state_lock(server->service); + cnt = scnprintf(buf, PAGE_SIZE, "%d\n", server->started); + vs_service_state_unlock(server->service); + + return cnt; +} + +static DEVICE_ATTR(start, S_IWUSR | S_IRUSR, vs_block_server_start_show, + vs_block_server_start_store); +static DEVICE_ATTR(readonly, S_IWUSR | S_IRUSR, vs_block_server_readonly_show, + vs_block_server_readonly_store); + +static struct attribute *vs_block_server_dev_attrs[] = { + &dev_attr_start.attr, + &dev_attr_readonly.attr, + NULL, +}; + +static const struct attribute_group vs_block_server_attr_group = { + .attrs = vs_block_server_dev_attrs +}; + +/* + * Invoked by vs_server_block_handle_req_open() after receiving open + * requests to perform server specific initialisations + * + * The "delayed start" feature can be enforced here + */ +static vs_server_response_type_t +vs_block_server_open(struct vs_server_block_state * _state) +{ + struct block_server *server = state_to_block_server(_state); + + return (server->started) ? VS_SERVER_RESP_SUCCESS : + VS_SERVER_RESP_EXPLICIT_COMPLETE; +} + +static int +vs_block_server_complete_req_read(struct block_server_request *req) +{ + struct block_server *server = req->server; + struct vs_server_block_state *state = &server->server; + int err = -EIO; + + if (req->op_err) { + err = req->op_err; + dev_dbg(&server->service->dev, + "read nack, err %d sector 0x%llx num 0x%x\n", + err, vs_req_sector_index(req), + vs_req_num_sectors(server, req)); + + if (req->mbuf) + vs_server_block_io_free_ack_read(state, &req->pbuf, + req->mbuf); + + err = vs_server_block_io_send_nack_read(state, req->tagid, + block_server_linux_to_vs_error(err), + GFP_KERNEL); + } else { + if (req->bounced && !req->mbuf) { + req->mbuf = vs_server_block_io_alloc_ack_read( + &server->server, &req->pbuf, + GFP_KERNEL); + if (IS_ERR(req->mbuf)) { + err = PTR_ERR(req->mbuf); + req->mbuf = NULL; + } + } + + if (req->bounced && req->mbuf) { + int i; + struct bio_vec *bv; + void *data = req->pbuf.data; + + if (vs_pbuf_resize(&req->pbuf, req->size) < 0) { + bio_io_error(&req->bio); + return 0; + } + + bio_for_each_segment_all(bv, &req->bio, i) { + memcpy(data, page_address(bv->bv_page) + + bv->bv_offset, bv->bv_len); + data += bv->bv_len; + __free_page(bv->bv_page); + } + req->bounced = false; + } + + if (req->mbuf) { + dev_vdbg(&server->service->dev, + "read ack, sector 0x%llx num 0x%x\n", + vs_req_sector_index(req), + vs_req_num_sectors(server, req)); + + err = vs_server_block_io_send_ack_read(state, + req->tagid, req->pbuf, req->mbuf); + + if (err && (err != -ENOBUFS)) { + vs_server_block_io_free_ack_read(state, + &req->pbuf, req->mbuf); + req->mbuf = NULL; + } + } else { + WARN_ON(!err || !req->bounced); + } + } + + if (err && (err != -ENOBUFS)) + dev_dbg(&server->service->dev, + "error %d sending read reply\n", err); + else if (err == -ENOBUFS) + dev_vdbg(&server->service->dev, "out of quota, will retry\n"); + + return err; +} + +static int +vs_block_server_complete_req_write(struct block_server_request *req) +{ + struct block_server *server = req->server; + struct vs_server_block_state *state = &server->server; + int err; + + WARN_ON(req->mbuf); + + if (req->op_err) { + dev_dbg(&server->service->dev, + "write nack, err %d sector 0x%llx num 0x%x\n", + req->op_err, vs_req_sector_index(req), + vs_req_num_sectors(server, req)); + + err = vs_server_block_io_send_nack_write(state, req->tagid, + block_server_linux_to_vs_error(req->op_err), + GFP_KERNEL); + } else { + dev_vdbg(&server->service->dev, + "write ack, sector 0x%llx num 0x%x\n", + vs_req_sector_index(req), + vs_req_num_sectors(server, req)); + + err = vs_server_block_io_send_ack_write(state, req->tagid, + GFP_KERNEL); + } + + if (err && (err != -ENOBUFS)) + dev_dbg(&server->service->dev, + "error %d sending write reply\n", err); + else if (err == -ENOBUFS) + dev_vdbg(&server->service->dev, "out of quota, will retry\n"); + + return err; +} + +static int vs_block_server_complete_req(struct block_server *server, + struct block_server_request *req) +{ + int err; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0) + req->bio.bi_iter.bi_idx = 0; +#else + req->bio.bi_idx = 0; +#endif + if (!vs_state_lock_safe(&server->server)) + return -ENOLINK; + + if (bio_data_dir(&req->bio) == WRITE) + err = vs_block_server_complete_req_write(req); + else + err = vs_block_server_complete_req_read(req); + + vs_state_unlock(&server->server); + + if (err == -ENOBUFS) + dev_vdbg(&server->service->dev, "bio %pK response out of quota, will retry\n", &req->bio); + + return err; +} + +static void vs_block_server_complete_requests_work(struct work_struct *work) +{ + struct block_server *server = container_of(work, struct block_server, + completed_req_work); + struct block_server_request *req; + + vs_service_send_batch_start(server->service, false); + + /* + * Send ack/nack responses for each completed request. If a request + * cannot be sent because we are over-quota then this function will + * return with a non-empty list, and the tx_ready handler will + * reschedule us when we are back under quota. In all other cases + * this function will return with an empty list. + */ + spin_lock_irq(&server->completed_req_lock); + while (!list_empty(&server->completed_req_queue)) { + int err; + req = list_first_entry(&server->completed_req_queue, + struct block_server_request, list); + dev_vdbg(&server->service->dev, "complete bio %pK\n", &req->bio); + list_del(&req->list); + spin_unlock_irq(&server->completed_req_lock); + + err = vs_block_server_complete_req(server, req); + if (err == -ENOBUFS) { + dev_vdbg(&server->service->dev, "defer bio %pK\n", &req->bio); + /* + * Couldn't send the completion; re-queue the request + * and exit. We'll start again when more quota becomes + * available. + */ + spin_lock_irq(&server->completed_req_lock); + list_add_tail(&req->list, + &server->completed_req_queue); + break; + } + + dev_vdbg(&server->service->dev, "free bio %pK err %d\n", &req->bio, err); + bio_put(&req->bio); + + spin_lock_irq(&server->completed_req_lock); + } + spin_unlock_irq(&server->completed_req_lock); + + vs_service_send_batch_end(server->service, true); +} + +static int vs_block_server_tx_ready(struct vs_server_block_state *state) +{ + struct block_server *server = state_to_block_server(state); + + schedule_work(&server->completed_req_work); + + return 0; +} + +static bool vs_block_can_map_pbuf(struct request_queue *q, + struct vs_pbuf *pbuf, size_t size) +{ + /* The pbuf must satisfy the driver's alignment requirements. */ + if (!blk_rq_aligned(q, (unsigned long)pbuf->data, size)) + return false; + + /* + * bios can only contain pages. Sometime the pbuf is in an IO region + * that has no struct page (e.g. a channel primary buffer), in which + * case we can't map it into a bio. + */ + /* FIXME: Redmine issue #930 - philip. */ + if (!pfn_valid(__pa(pbuf->data) >> PAGE_SHIFT)) + return false; + + return true; +} + +static int vs_block_bio_map_pbuf(struct bio *bio, struct vs_pbuf *pbuf) +{ + int offset = offset_in_page((unsigned long)pbuf->data); + void *ptr = pbuf->data; + int size = pbuf->size; + + while (size > 0) { + unsigned bytes = min_t(unsigned, PAGE_SIZE - offset, size); + + if (bio_add_page(bio, virt_to_page(ptr), bytes, + offset) < bytes) + return -EIO; + + ptr += bytes; + size -= bytes; + offset = 0; + } + + return 0; +} + +/* Read request handling */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) +static void vs_block_server_read_done(struct bio *bio, int err) +#else +static void vs_block_server_read_done(struct bio *bio) +#endif +{ + unsigned long flags; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0) + int err = bio->bi_error; +#endif + struct block_server_request *req = container_of(bio, + struct block_server_request, bio); + struct block_server *server = req->server; + req->op_err = err; + + spin_lock_irqsave(&server->completed_req_lock, flags); + if (req->mbuf) + list_add(&req->list, &server->completed_req_queue); + else + list_add_tail(&req->list, &server->completed_req_queue); + spin_unlock_irqrestore(&server->completed_req_lock, flags); + + if (req->submitted && atomic_dec_and_test(&server->submitted_req_count)) + wake_up_all(&server->submitted_req_wq); + + schedule_work(&server->completed_req_work); +} + +/* + * TODO: this may need to split and chain the bio if it exceeds the physical + * segment limit of the device. Not clear whose responsibility that is; queue + * might do it for us (if there is one) + */ +#define vs_block_make_request(bio) generic_make_request(bio) + +static int vs_block_submit_read(struct block_server *server, + struct block_server_request *req, gfp_t gfp) +{ + struct request_queue *q = bdev_get_queue(server->bdev); + struct bio *bio = &req->bio; + int size = req->size; + int err = 0; + + if (req->mbuf && vs_block_can_map_pbuf(q, &req->pbuf, size)) { + /* + * The mbuf is valid and the driver can directly access the + * pbuf, so we don't need a bounce buffer. Map the pbuf + * directly into the bio. + */ + if (vs_pbuf_resize(&req->pbuf, size) < 0) + err = -EIO; + if (!err) + err = vs_block_bio_map_pbuf(bio, &req->pbuf); + } else { + /* We need a bounce buffer. First set up the bvecs. */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0) + bio->bi_iter.bi_size = size; +#else + bio->bi_size = size; +#endif + + while (size > 0) { + struct bio_vec *bvec = &bio->bi_io_vec[bio->bi_vcnt]; + + BUG_ON(bio->bi_vcnt >= bio->bi_max_vecs); + + bvec->bv_page = NULL; /* Allocated below */ + bvec->bv_len = min_t(unsigned, PAGE_SIZE, size); + bvec->bv_offset = 0; + + bio->bi_vcnt++; + size -= bvec->bv_len; + } + + err = bio_alloc_pages(bio, gfp); + if (!err) { + blk_recount_segments(q, bio); + req->bounced = true; + } + } + + if (err) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0) + bio->bi_error = err; + bio_endio(bio); +#else + bio_endio(bio, err); +#endif + } else { + dev_vdbg(&server->service->dev, + "submit read req sector %#llx count %#x\n", + vs_req_sector_index(req), + vs_req_num_sectors(server, req)); + req->submitted = true; + atomic_inc(&server->submitted_req_count); + vs_block_make_request(bio); + } + + return 0; +} + +static int vs_block_server_io_req_read(struct vs_server_block_state *state, + u32 tagid, u64 sector_index, u32 num_sects, bool nodelay, + bool flush) +{ + struct block_server *server = state_to_block_server(state); + struct bio *bio; + struct block_server_request *req; + unsigned size = num_sects * server->sector_size; + unsigned op_flags = 0; + + /* + * This nr_pages calculation assumes that the pbuf data is offset from + * the start of the size-aligned message buffer by more than 0 but + * less than one sector, which is always true for the current message + * layout generated by mill when we assume 512-byte sectors. + */ + unsigned nr_pages = 1 + (size >> PAGE_SHIFT); + + bio = bio_alloc_bioset(GFP_KERNEL, nr_pages, server->bioset); + if (!bio) + return -ENOMEM; + dev_vdbg(&server->service->dev, "alloc r bio %pK\n", bio); + req = container_of(bio, struct block_server_request, bio); + + req->server = server; + req->tagid = tagid; + req->op_err = 0; + req->mbuf = NULL; + req->size = size; + req->bounced = false; + req->submitted = false; + + if (flush) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,8,0) + op_flags |= REQ_PREFLUSH; +#else + op_flags |= REQ_FLUSH; +#endif + } + if (nodelay) { + op_flags |= REQ_SYNC; + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0) + bio->bi_iter.bi_sector = (sector_t)sector_index; +#else + bio->bi_sector = (sector_t)sector_index; +#endif + bio->bi_bdev = server->bdev; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0) + bio_set_op_attrs(bio, REQ_OP_READ, op_flags); +#else + bio->bi_rw = READ | op_flags; +#endif + bio->bi_end_io = vs_block_server_read_done; + + req->mbuf = vs_server_block_io_alloc_ack_read(state, &req->pbuf, + GFP_KERNEL); + if (IS_ERR(req->mbuf) && (PTR_ERR(req->mbuf) == -ENOBUFS)) { + /* Fall back to a bounce buffer */ + req->mbuf = NULL; + } else if (IS_ERR(req->mbuf)) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0) + bio->bi_error = PTR_ERR(req->mbuf); + bio_endio(bio); +#else + bio_endio(bio, PTR_ERR(req->mbuf)); +#endif + return 0; + } + + return vs_block_submit_read(server, req, GFP_KERNEL); +} + +/* Write request handling */ +static int vs_block_submit_bounced_write(struct block_server *server, + struct block_server_request *req, gfp_t gfp) +{ + struct bio *bio = &req->bio; + void *data = req->pbuf.data; + struct bio_vec *bv; + int i; + + if (bio_alloc_pages(bio, gfp | __GFP_NOWARN) < 0) + return -ENOMEM; + blk_recount_segments(bdev_get_queue(server->bdev), bio); + req->bounced = true; + + /* Copy all the data into the bounce buffer */ + bio_for_each_segment_all(bv, bio, i) { + memcpy(page_address(bv->bv_page) + bv->bv_offset, data, + bv->bv_len); + data += bv->bv_len; + } + + vs_server_block_io_free_req_write(&server->server, &req->pbuf, + req->mbuf); + req->mbuf = NULL; + + dev_vdbg(&server->service->dev, + "submit bounced write req sector %#llx count %#x\n", + vs_req_sector_index(req), + vs_req_num_sectors(server, req)); + req->submitted = true; + atomic_inc(&server->submitted_req_count); + vs_block_make_request(bio); + + return 0; +} + +static void vs_block_server_write_bounce_work(struct work_struct *work) +{ + struct block_server *server = container_of(work, struct block_server, + bounce_req_work); + struct block_server_request *req; + + spin_lock(&server->bounce_req_lock); + while (!list_empty(&server->bounce_req_queue)) { + req = list_first_entry(&server->bounce_req_queue, + struct block_server_request, list); + dev_vdbg(&server->service->dev, "write bio %pK\n", &req->bio); + list_del(&req->list); + spin_unlock(&server->bounce_req_lock); + + if (vs_block_submit_bounced_write(server, req, + GFP_KERNEL) == -ENOMEM) { + spin_lock(&server->bounce_req_lock); + list_add(&req->list, &server->bounce_req_queue); + spin_unlock(&server->bounce_req_lock); + schedule_work(work); + return; + } + + spin_lock(&server->bounce_req_lock); + } + spin_unlock(&server->bounce_req_lock); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) +static void vs_block_server_write_done(struct bio *bio, int err) +#else +static void vs_block_server_write_done(struct bio *bio) +#endif +{ + unsigned long flags; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0) + int err = bio->bi_error; +#endif + struct block_server_request *req = container_of(bio, + struct block_server_request, bio); + struct block_server *server = req->server; + + if (req->bounced) { + int i; + struct bio_vec *bv; + bio_for_each_segment_all(bv, bio, i) + __free_page(bv->bv_page); + } else if (req->mbuf) { + vs_server_block_io_free_req_write(&server->server, &req->pbuf, + req->mbuf); + req->mbuf = NULL; + } + + if (req->submitted && atomic_dec_and_test(&server->submitted_req_count)) + wake_up_all(&server->submitted_req_wq); + + req->op_err = err; + + spin_lock_irqsave(&server->completed_req_lock, flags); + list_add_tail(&req->list, &server->completed_req_queue); + spin_unlock_irqrestore(&server->completed_req_lock, flags); + + schedule_work(&server->completed_req_work); +} + +static int vs_block_server_io_req_write(struct vs_server_block_state *state, + u32 tagid, u64 sector_index, u32 num_sects, bool nodelay, + bool flush, bool commit, struct vs_pbuf pbuf, struct vs_mbuf *mbuf) +{ + struct block_server *server = state_to_block_server(state); + struct request_queue *q = bdev_get_queue(server->bdev); + struct bio *bio; + struct block_server_request *req; + unsigned long data = (unsigned long)pbuf.data; + unsigned long start = data >> PAGE_SHIFT; + unsigned long end = (data + pbuf.size + PAGE_SIZE - 1) >> PAGE_SHIFT; + int err; + unsigned op_flags = 0; + + bio = bio_alloc_bioset(GFP_KERNEL, end - start, server->bioset); + if (!bio) + return -ENOMEM; + dev_vdbg(&server->service->dev, "alloc w bio %pK\n", bio); + req = container_of(bio, struct block_server_request, bio); + + req->server = server; + req->tagid = tagid; + req->op_err = 0; + req->mbuf = mbuf; + req->pbuf = pbuf; + req->size = server->sector_size * num_sects; + req->bounced = false; + req->submitted = false; + + if (flush) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,8,0) + op_flags |= REQ_PREFLUSH; +#else + op_flags |= REQ_FLUSH; +#endif + } + if (commit) { + op_flags |= REQ_FUA; + } + if (nodelay) { + op_flags |= REQ_SYNC; + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0) + bio->bi_iter.bi_sector = (sector_t)sector_index; +#else + bio->bi_sector = (sector_t)sector_index; +#endif + bio->bi_bdev = server->bdev; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0) + bio_set_op_attrs(bio, REQ_OP_WRITE, op_flags); +#else + bio->bi_rw = WRITE | op_flags; +#endif + bio->bi_end_io = vs_block_server_write_done; + + if (pbuf.size < req->size) { + err = -EINVAL; + goto fail_bio; + } + if (WARN_ON(pbuf.size > req->size)) + pbuf.size = req->size; + + if (state->readonly) { + err = -EROFS; + goto fail_bio; + } + + if (!vs_block_can_map_pbuf(q, &req->pbuf, req->pbuf.size)) { + /* We need a bounce buffer. First set up the bvecs. */ + int size = pbuf.size; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0) + bio->bi_iter.bi_size = size; +#else + bio->bi_size = size; +#endif + + while (size > 0) { + struct bio_vec *bvec = &bio->bi_io_vec[bio->bi_vcnt]; + + BUG_ON(bio->bi_vcnt >= bio->bi_max_vecs); + + bvec->bv_page = NULL; /* Allocated later */ + bvec->bv_len = min_t(unsigned, PAGE_SIZE, size); + bvec->bv_offset = 0; + + bio->bi_vcnt++; + size -= bvec->bv_len; + } + + /* + * Defer the rest so we don't have to hold the state lock + * during alloc_page & memcpy + */ + spin_lock(&server->bounce_req_lock); + list_add_tail(&req->list, &server->bounce_req_queue); + spin_unlock(&server->bounce_req_lock); + schedule_work(&server->bounce_req_work); + + return 0; + } + + /* No bounce needed; map the pbuf directly. */ + err = vs_block_bio_map_pbuf(bio, &pbuf); + if (err < 0) + goto fail_bio; + + dev_vdbg(&server->service->dev, + "submit direct write req sector %#llx count %#x\n", + vs_req_sector_index(req), + vs_req_num_sectors(server, req)); + req->submitted = true; + atomic_inc(&server->submitted_req_count); + vs_block_make_request(bio); + + return 0; + +fail_bio: +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0) + bio->bi_error = err; + bio_endio(bio); +#else + bio_endio(bio, err); +#endif + return 0; +} + +static struct block_device * +vs_block_server_find_by_name(struct block_server *server) +{ + struct block_device *bdev = NULL; + struct class_dev_iter iter; + struct device *dev; + + class_dev_iter_init(&iter, &block_class, NULL, NULL); + while (1) { + dev = class_dev_iter_next(&iter); + if (!dev) + break; + + if (strcmp(dev_name(dev), server->service->name) == 0) { + bdev = blkdev_get_by_dev(dev->devt, + VS_BLOCK_BLKDEV_DEFAULT_MODE, NULL); + if (!IS_ERR_OR_NULL(bdev)) + break; + } + } + class_dev_iter_exit(&iter); + + if (!dev || IS_ERR_OR_NULL(bdev)) + return ERR_PTR(-ENODEV); + + dev_dbg(&server->service->dev, "Attached to block device %s (%d:%d)\n", + dev_name(dev), MAJOR(dev->devt), MINOR(dev->devt)); + return bdev; +} + +static struct block_device * +vs_block_server_find_by_path(struct block_server *server, const char *base_path) +{ + struct block_device *bdev; + char *bdev_path; + + bdev_path = kasprintf(GFP_KERNEL, "%s/%s", base_path, + server->service->name); + if (!bdev_path) + return ERR_PTR(-ENOMEM); + + bdev = blkdev_get_by_path(bdev_path, VS_BLOCK_BLKDEV_DEFAULT_MODE, + NULL); + dev_dbg(&server->service->dev, "Attached to block device %s\n", + bdev_path); + + kfree(bdev_path); + + if (!bdev) + return ERR_PTR(-ENODEV); + return bdev; +} + +static struct block_device * +vs_block_server_attach_block_device(struct block_server *server) +{ + const char *paths[] = { + "/dev", + "/dev/block", + "/dev/mapper", + "/dev/disk/by-partlabel", + "/dev/disk/by-label", + "/dev/disk/by-partuuid", + "/dev/disk/by-uuid" + }; + struct block_device *bdev; + int i; + + /* + * Try first to look the block device up by path. This is done because + * the name exposed to user-space in /dev/ is not necessarily the name + * being used inside the kernel for the device. + */ + for (i = 0; i < ARRAY_SIZE(paths); i++) { + bdev = vs_block_server_find_by_path(server, paths[i]); + if (!IS_ERR(bdev)) + break; + } + if (i == ARRAY_SIZE(paths)) { + /* + * Couldn't find the block device in any of the usual places. + * Try to match it against the kernel's device name. If the + * name of the service and the name of a device in the block + * class match then attempt to look the block device up by the + * dev_t (major/minor) value. + */ + bdev = vs_block_server_find_by_name(server); + } + if (IS_ERR(bdev)) + return bdev; + + server->sector_size = VS_BLOCK_BLK_DEF_SECTOR_SIZE; + server->server.segment_size = round_down( + vs_service_max_mbuf_size(server->service) - + sizeof(vs_message_id_t), server->sector_size); + server->server.sector_size = server->sector_size; + server->server.device_sectors = bdev->bd_part->nr_sects; + if (bdev_read_only(bdev)) + server->server.readonly = true; + server->server.flushable = true; + server->server.committable = true; + + return bdev; +} + +static struct vs_server_block_state * +vs_block_server_alloc(struct vs_service_device *service) +{ + struct block_server *server; + int err; + + server = kzalloc(sizeof(*server), GFP_KERNEL); + if (!server) + return NULL; + + server->service = service; + server->started = false; + INIT_LIST_HEAD(&server->bounce_req_queue); + INIT_WORK(&server->bounce_req_work, vs_block_server_write_bounce_work); + spin_lock_init(&server->bounce_req_lock); + atomic_set(&server->submitted_req_count, 0); + init_waitqueue_head(&server->submitted_req_wq); + INIT_LIST_HEAD(&server->completed_req_queue); + INIT_WORK(&server->completed_req_work, + vs_block_server_complete_requests_work); + spin_lock_init(&server->completed_req_lock); + + server->bdev = vs_block_server_attach_block_device(server); + if (IS_ERR(server->bdev)) { + dev_err(&server->service->dev, + "No appropriate block device was found to satisfy the service name %s - error %ld\n", + server->service->name, PTR_ERR(server->bdev)); + goto fail_attach_device; + } + + dev_set_drvdata(&service->dev, &server->server); + + err = sysfs_create_group(&service->dev.kobj, + &vs_block_server_attr_group); + if (err) { + dev_err(&service->dev, + "Failed to create attribute group for service %s\n", + service->name); + goto fail_create_group; + } + + /* + * We know the upper bound on simultaneously active bios (i.e. the + * smaller of the in quota, and the sum of the read and write command + * tag limits), so we can pre-allocate that many, and hopefully never + * fail to allocate one in a request handler. + * + * However, allocation may fail if the number of pages (and thus + * bvecs) in a request exceeds BIO_INLINE_VECS (which is hard-coded to + * 4 in all mainline kernels). That possibility is the only reason we + * can't enable rx_atomic for this driver. + */ + server->bioset = bioset_create(min_t(unsigned, service->recv_quota, + VSERVICE_BLOCK_IO_READ_MAX_PENDING + + VSERVICE_BLOCK_IO_WRITE_MAX_PENDING), + offsetof(struct block_server_request, bio)); + if (!server->bioset) { + dev_err(&service->dev, + "Failed to allocate bioset for service %s\n", + service->name); + goto fail_create_bioset; + } + + dev_dbg(&service->dev, "New block server %pK\n", server); + + return &server->server; + +fail_create_bioset: + sysfs_remove_group(&server->service->dev.kobj, + &vs_block_server_attr_group); +fail_create_group: + dev_set_drvdata(&service->dev, NULL); + blkdev_put(server->bdev, VS_BLOCK_BLKDEV_DEFAULT_MODE); +fail_attach_device: + kfree(server); + + return NULL; +} + +static void vs_block_server_release(struct vs_server_block_state *state) +{ + struct block_server *server = state_to_block_server(state); + + cancel_work_sync(&server->bounce_req_work); + cancel_work_sync(&server->completed_req_work); + + blkdev_put(server->bdev, VS_BLOCK_BLKDEV_DEFAULT_MODE); + + sysfs_remove_group(&server->service->dev.kobj, + &vs_block_server_attr_group); + + bioset_free(server->bioset); + + kfree(server); +} + +static struct vs_server_block block_server_driver = { + .alloc = vs_block_server_alloc, + .release = vs_block_server_release, + .open = vs_block_server_open, + .closed = vs_block_server_closed, + .tx_ready = vs_block_server_tx_ready, + .io = { + .req_read = vs_block_server_io_req_read, + .req_write = vs_block_server_io_req_write, + }, + + /* Large default quota for batching read/write commands */ + .in_quota_best = 32, + .out_quota_best = 32, +}; + +static int __init vs_block_server_init(void) +{ + return vservice_block_server_register(&block_server_driver, + "block_server_driver"); +} + +static void __exit vs_block_server_exit(void) +{ + vservice_block_server_unregister(&block_server_driver); +} + +module_init(vs_block_server_init); +module_exit(vs_block_server_exit); + +MODULE_DESCRIPTION("OKL4 Virtual Services Block Server Driver"); +MODULE_AUTHOR("Open Kernel Labs, Inc"); diff --git a/drivers/vservices/protocol/Kconfig b/drivers/vservices/protocol/Kconfig index 6dd280d2e88e..e0f2798c8a0d 100644 --- a/drivers/vservices/protocol/Kconfig +++ b/drivers/vservices/protocol/Kconfig @@ -5,6 +5,23 @@ if VSERVICES_SERVER || VSERVICES_CLIENT menu "Protocol drivers" +config VSERVICES_PROTOCOL_BLOCK + bool + +config VSERVICES_PROTOCOL_BLOCK_SERVER + tristate "Block server protocol" + depends on VSERVICES_SUPPORT && VSERVICES_SERVER + select VSERVICES_PROTOCOL_BLOCK + help + This option adds support for Virtual Services block protocol server. + +config VSERVICES_PROTOCOL_BLOCK_CLIENT + tristate "Block client protocol" + depends on VSERVICES_SUPPORT && VSERVICES_CLIENT + select VSERVICES_PROTOCOL_BLOCK + help + This option adds support for Virtual Services block protocol client. + config VSERVICES_PROTOCOL_SERIAL bool diff --git a/drivers/vservices/protocol/Makefile b/drivers/vservices/protocol/Makefile index 10b99b554816..0c714e05e4c7 100644 --- a/drivers/vservices/protocol/Makefile +++ b/drivers/vservices/protocol/Makefile @@ -1,4 +1,5 @@ # This is a autogenerated Makefile for vservice-linux-stacks obj-$(CONFIG_VSERVICES_SUPPORT) += core/ +obj-$(CONFIG_VSERVICES_PROTOCOL_BLOCK) += block/ obj-$(CONFIG_VSERVICES_PROTOCOL_SERIAL) += serial/ diff --git a/drivers/vservices/protocol/block/Makefile b/drivers/vservices/protocol/block/Makefile new file mode 100644 index 000000000000..325b57e390e2 --- /dev/null +++ b/drivers/vservices/protocol/block/Makefile @@ -0,0 +1,7 @@ +ccflags-y += -Werror + +obj-$(CONFIG_VSERVICES_PROTOCOL_BLOCK_SERVER) += vservices_protocol_block_server.o +vservices_protocol_block_server-objs = server.o + +obj-$(CONFIG_VSERVICES_PROTOCOL_BLOCK_CLIENT) += vservices_protocol_block_client.o +vservices_protocol_block_client-objs = client.o diff --git a/drivers/vservices/protocol/block/client.c b/drivers/vservices/protocol/block/client.c new file mode 100644 index 000000000000..702a30a82a9d --- /dev/null +++ b/drivers/vservices/protocol/block/client.c @@ -0,0 +1,1186 @@ + +/* + * Copyright (c) 2012-2018 General Dynamics + * Copyright (c) 2014 Open Kernel Labs, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + + /* + * This is the generated code for the block client protocol handling. + */ +#include +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0) +#include +#endif + +#include +#include +#include +#include +#include +#include + +#include "../../transport.h" + +#define VS_MBUF_SIZE(mbuf) mbuf->size +#define VS_MBUF_DATA(mbuf) mbuf->data +#define VS_STATE_SERVICE_PTR(state) state->service + +static int _vs_client_block_req_open(struct vs_client_block_state *_state); + +/*** Linux driver model integration ***/ +struct vs_block_client_driver { + struct vs_client_block *client; + struct list_head list; + struct vs_service_driver vsdrv; +}; + +#define to_client_driver(d) \ + container_of(d, struct vs_block_client_driver, vsdrv) + +static void reset_nack_requests(struct vs_service_device *service) +{ + + struct vs_client_block_state *state = dev_get_drvdata(&service->dev); + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + struct vs_client_block *client __maybe_unused = + to_client_driver(vsdrv)->client; + + int i __maybe_unused; + + /* Clear out pending read commands */ + for_each_set_bit(i, state->state.io.read_bitmask, + VSERVICE_BLOCK_IO_READ_MAX_PENDING) { + void *tag = state->state.io.read_tags[i]; + + if (client->io.nack_read) + client->io.nack_read(state, tag, + VSERVICE_BLOCK_SERVICE_RESET); + + __clear_bit(i, state->state.io.read_bitmask); + } + + /* Clear out pending write commands */ + for_each_set_bit(i, state->state.io.write_bitmask, + VSERVICE_BLOCK_IO_WRITE_MAX_PENDING) { + void *tag = state->state.io.write_tags[i]; + + if (client->io.nack_write) + client->io.nack_write(state, tag, + VSERVICE_BLOCK_SERVICE_RESET); + + __clear_bit(i, state->state.io.write_bitmask); + } + +} + +static void block_handle_start(struct vs_service_device *service) +{ + + struct vs_client_block_state *state = dev_get_drvdata(&service->dev); + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + struct vs_client_block *client __maybe_unused = + to_client_driver(vsdrv)->client; + + vs_service_state_lock(service); + state->state = VSERVICE_BLOCK_RESET_STATE; + + _vs_client_block_req_open(state); + + vs_service_state_unlock(service); +} + +static void block_handle_reset(struct vs_service_device *service) +{ + + struct vs_client_block_state *state = dev_get_drvdata(&service->dev); + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + struct vs_client_block *client __maybe_unused = + to_client_driver(vsdrv)->client; + + vs_service_state_lock(service); + if (!VSERVICE_BASE_STATE_IS_RUNNING(state->state.base)) { + vs_service_state_unlock(service); + return; + } + state->state.base = VSERVICE_BASE_RESET_STATE; + reset_nack_requests(service); + if (client->closed) + client->closed(state); + + state->state = VSERVICE_BLOCK_RESET_STATE; + + vs_service_state_unlock(service); +} + +static void block_handle_start_bh(struct vs_service_device *service) +{ + + struct vs_client_block_state *state = dev_get_drvdata(&service->dev); + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + struct vs_client_block *client __maybe_unused = + to_client_driver(vsdrv)->client; + + vs_service_state_lock_bh(service); + state->state = VSERVICE_BLOCK_RESET_STATE; + + _vs_client_block_req_open(state); + + vs_service_state_unlock_bh(service); +} + +static void block_handle_reset_bh(struct vs_service_device *service) +{ + + struct vs_client_block_state *state = dev_get_drvdata(&service->dev); + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + struct vs_client_block *client __maybe_unused = + to_client_driver(vsdrv)->client; + + vs_service_state_lock_bh(service); + if (!VSERVICE_BASE_STATE_IS_RUNNING(state->state.base)) { + vs_service_state_unlock_bh(service); + return; + } + state->state.base = VSERVICE_BASE_RESET_STATE; + reset_nack_requests(service); + if (client->closed) + client->closed(state); + + state->state = VSERVICE_BLOCK_RESET_STATE; + + vs_service_state_unlock_bh(service); +} + +static int block_client_probe(struct vs_service_device *service); +static int block_client_remove(struct vs_service_device *service); +static int block_handle_message(struct vs_service_device *service, + struct vs_mbuf *_mbuf); +static void block_handle_notify(struct vs_service_device *service, + uint32_t flags); +static void block_handle_start(struct vs_service_device *service); +static void block_handle_start_bh(struct vs_service_device *service); +static void block_handle_reset(struct vs_service_device *service); +static void block_handle_reset_bh(struct vs_service_device *service); +static int block_handle_tx_ready(struct vs_service_device *service); + +int __vservice_block_client_register(struct vs_client_block *client, + const char *name, struct module *owner) +{ + int ret; + struct vs_block_client_driver *driver; + + if (client->tx_atomic && !client->rx_atomic) + return -EINVAL; + + driver = kzalloc(sizeof(*driver), GFP_KERNEL); + if (!driver) { + ret = -ENOMEM; + goto fail_alloc_driver; + } + + client->driver = &driver->vsdrv; + driver->client = client; + + driver->vsdrv.protocol = VSERVICE_BLOCK_PROTOCOL_NAME; + + driver->vsdrv.is_server = false; + driver->vsdrv.rx_atomic = client->rx_atomic; + driver->vsdrv.tx_atomic = client->tx_atomic; + + driver->vsdrv.probe = block_client_probe; + driver->vsdrv.remove = block_client_remove; + driver->vsdrv.receive = block_handle_message; + driver->vsdrv.notify = block_handle_notify; + driver->vsdrv.start = client->tx_atomic ? + block_handle_start_bh : block_handle_start; + driver->vsdrv.reset = client->tx_atomic ? + block_handle_reset_bh : block_handle_reset; + driver->vsdrv.tx_ready = block_handle_tx_ready; + driver->vsdrv.out_notify_count = 0; + driver->vsdrv.in_notify_count = 0; + driver->vsdrv.driver.name = name; + driver->vsdrv.driver.owner = owner; + driver->vsdrv.driver.bus = &vs_client_bus_type; + + ret = driver_register(&driver->vsdrv.driver); + + if (ret) { + goto fail_driver_register; + } + + return 0; + + fail_driver_register: + client->driver = NULL; + kfree(driver); + fail_alloc_driver: + return ret; +} + +EXPORT_SYMBOL(__vservice_block_client_register); + +int vservice_block_client_unregister(struct vs_client_block *client) +{ + struct vs_block_client_driver *driver; + + if (!client->driver) + return 0; + + driver = to_client_driver(client->driver); + driver_unregister(&driver->vsdrv.driver); + + client->driver = NULL; + kfree(driver); + + return 0; +} + +EXPORT_SYMBOL(vservice_block_client_unregister); + +static int block_client_probe(struct vs_service_device *service) +{ + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + struct vs_client_block *client = to_client_driver(vsdrv)->client; + struct vs_client_block_state *state; + + state = client->alloc(service); + if (!state) + return -ENOMEM; + else if (IS_ERR(state)) + return PTR_ERR(state); + + state->service = vs_get_service(service); + state->state = VSERVICE_BLOCK_RESET_STATE; + + dev_set_drvdata(&service->dev, state); + + return 0; +} + +static int block_client_remove(struct vs_service_device *service) +{ + struct vs_client_block_state *state = dev_get_drvdata(&service->dev); + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + struct vs_client_block *client = to_client_driver(vsdrv)->client; + + state->released = true; + dev_set_drvdata(&service->dev, NULL); + client->release(state); + + vs_put_service(service); + + return 0; +} + +static int block_handle_tx_ready(struct vs_service_device *service) +{ + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + struct vs_client_block *client = to_client_driver(vsdrv)->client; + struct vs_client_block_state *state = dev_get_drvdata(&service->dev); + + if (!VSERVICE_BASE_STATE_IS_RUNNING(state->state.base)) + return 0; + + if (client->tx_ready) + client->tx_ready(state); + + return 0; +} + +static int _vs_client_block_req_open(struct vs_client_block_state *_state) +{ + struct vs_mbuf *_mbuf; + + const size_t _msg_size = sizeof(vs_message_id_t) + 0UL; + + struct vs_service_driver *vsdrv = + to_vs_service_driver(VS_STATE_SERVICE_PTR(_state)->dev.driver); + __maybe_unused struct vs_client_block *_client = + to_client_driver(vsdrv)->client; + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_CLOSED: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + + _mbuf = + vs_service_alloc_mbuf(VS_STATE_SERVICE_PTR(_state), _msg_size, + vs_service_has_atomic_rx(VS_STATE_SERVICE_PTR + (_state)) ? + GFP_ATOMIC : GFP_KERNEL); + if (IS_ERR(_mbuf)) + return PTR_ERR(_mbuf); + if (!_mbuf) { + + WARN_ON_ONCE(1); + + return -ENOMEM; + } + + *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) = + VSERVICE_BLOCK_BASE_REQ_OPEN; + + _state->state.base.statenum = VSERVICE_BASE_STATE_CLOSED__OPEN; + + { + int err = vs_service_send(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (err) { + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: Error %d sending message on transport.\n", + __func__, __LINE__, err); + + return err; + } + } + + return 0; +} + +EXPORT_SYMBOL(_vs_client_block_req_open); +static int _vs_client_block_req_close(struct vs_client_block_state *_state) +{ + struct vs_mbuf *_mbuf; + + const size_t _msg_size = sizeof(vs_message_id_t) + 0UL; + + struct vs_service_driver *vsdrv = + to_vs_service_driver(VS_STATE_SERVICE_PTR(_state)->dev.driver); + __maybe_unused struct vs_client_block *_client = + to_client_driver(vsdrv)->client; + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_RUNNING: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + + _mbuf = + vs_service_alloc_mbuf(VS_STATE_SERVICE_PTR(_state), _msg_size, + vs_service_has_atomic_rx(VS_STATE_SERVICE_PTR + (_state)) ? + GFP_ATOMIC : GFP_KERNEL); + if (IS_ERR(_mbuf)) + return PTR_ERR(_mbuf); + if (!_mbuf) { + + WARN_ON_ONCE(1); + + return -ENOMEM; + } + + *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) = + VSERVICE_BLOCK_BASE_REQ_CLOSE; + + _state->state.base.statenum = VSERVICE_BASE_STATE_RUNNING__CLOSE; + + { + int err = vs_service_send(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (err) { + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: Error %d sending message on transport.\n", + __func__, __LINE__, err); + + return err; + } + } + + return 0; +} + +EXPORT_SYMBOL(_vs_client_block_req_close); +static int _vs_client_block_req_reopen(struct vs_client_block_state *_state) +{ + struct vs_mbuf *_mbuf; + + const size_t _msg_size = sizeof(vs_message_id_t) + 0UL; + + struct vs_service_driver *vsdrv = + to_vs_service_driver(VS_STATE_SERVICE_PTR(_state)->dev.driver); + __maybe_unused struct vs_client_block *_client = + to_client_driver(vsdrv)->client; + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_RUNNING: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + + _mbuf = + vs_service_alloc_mbuf(VS_STATE_SERVICE_PTR(_state), _msg_size, + vs_service_has_atomic_rx(VS_STATE_SERVICE_PTR + (_state)) ? + GFP_ATOMIC : GFP_KERNEL); + if (IS_ERR(_mbuf)) + return PTR_ERR(_mbuf); + if (!_mbuf) { + + WARN_ON_ONCE(1); + + return -ENOMEM; + } + + *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) = + VSERVICE_BLOCK_BASE_REQ_REOPEN; + + _state->state.base.statenum = VSERVICE_BASE_STATE_RUNNING__REOPEN; + + { + int err = vs_service_send(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (err) { + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: Error %d sending message on transport.\n", + __func__, __LINE__, err); + + return err; + } + } + + return 0; +} + +EXPORT_SYMBOL(_vs_client_block_req_reopen); +static int +block_base_handle_ack_open(const struct vs_client_block *_client, + struct vs_client_block_state *_state, + struct vs_mbuf *_mbuf) +{ + const size_t _expected_size = sizeof(vs_message_id_t) + 28UL; + + if (VS_MBUF_SIZE(_mbuf) < _expected_size) + return -EBADMSG; + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_CLOSED__OPEN: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + _state->state.base.statenum = VSERVICE_BASE_STATE_RUNNING; + _state->io.sector_size = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 4UL); + _state->io.segment_size = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 8UL); + _state->readonly = + *(bool *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 0UL); + _state->sector_size = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 4UL); + _state->segment_size = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 8UL); + _state->device_sectors = + *(uint64_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + + 12UL); + _state->flushable = + *(bool *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 20UL); + _state->committable = + *(bool *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 24UL); + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + _client->opened(_state); + return 0; + +} + +static int +block_base_handle_nack_open(const struct vs_client_block *_client, + struct vs_client_block_state *_state, + struct vs_mbuf *_mbuf) +{ + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_CLOSED__OPEN: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + _state->state.base.statenum = VSERVICE_BASE_STATE_CLOSED; + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + dev_err(&VS_STATE_SERVICE_PTR(_state)->dev, + "Open operation failed for device %s\n", + VS_STATE_SERVICE_PTR(_state)->name); + + return 0; + +} + +EXPORT_SYMBOL(block_base_handle_ack_open); +static int +block_base_handle_ack_close(const struct vs_client_block *_client, + struct vs_client_block_state *_state, + struct vs_mbuf *_mbuf) +{ + const size_t _expected_size = sizeof(vs_message_id_t) + 0UL; + + if (VS_MBUF_SIZE(_mbuf) < _expected_size) + return -EBADMSG; + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_RUNNING__CLOSE: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + _state->state.base.statenum = VSERVICE_BASE_STATE_CLOSED; + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + wake_up_all(&_state->service->quota_wq); + _client->closed(_state); + return 0; + +} + +static int +block_base_handle_nack_close(const struct vs_client_block *_client, + struct vs_client_block_state *_state, + struct vs_mbuf *_mbuf) +{ + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_RUNNING__CLOSE: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + _state->state.base.statenum = VSERVICE_BASE_STATE_RUNNING; + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + wake_up_all(&_state->service->quota_wq); + _client->closed(_state); + return 0; + +} + +EXPORT_SYMBOL(block_base_handle_ack_close); +static int +block_base_handle_ack_reopen(const struct vs_client_block *_client, + struct vs_client_block_state *_state, + struct vs_mbuf *_mbuf) +{ + const size_t _expected_size = sizeof(vs_message_id_t) + 0UL; + + if (VS_MBUF_SIZE(_mbuf) < _expected_size) + return -EBADMSG; + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_RUNNING__REOPEN: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + _state->state.base.statenum = VSERVICE_BASE__RESET; + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (_client->reopened) { + _client->reopened(_state); + return 0; + } + wake_up_all(&_state->service->quota_wq); + _client->closed(_state); + return _vs_client_block_req_open(_state); + +} + +static int +block_base_handle_nack_reopen(const struct vs_client_block *_client, + struct vs_client_block_state *_state, + struct vs_mbuf *_mbuf) +{ + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_RUNNING__REOPEN: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + return 0; + +} + +EXPORT_SYMBOL(block_base_handle_ack_reopen); +int vs_client_block_io_getbufs_ack_read(struct vs_client_block_state *_state, + struct vs_pbuf *data, + struct vs_mbuf *_mbuf) +{ + const vs_message_id_t _msg_id = VSERVICE_BLOCK_IO_ACK_READ; + const size_t _max_size = + sizeof(vs_message_id_t) + _state->io.segment_size + 8UL; + const size_t _min_size = _max_size - _state->io.segment_size; + size_t _exact_size; + + if (*(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) != _msg_id) + return -EINVAL; + if ((VS_MBUF_SIZE(_mbuf) > _max_size) + || (VS_MBUF_SIZE(_mbuf) < _min_size)) + return -EBADMSG; + + data->size = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 4UL); + data->data = + (uintptr_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 4UL + + sizeof(uint32_t)); + data->max_size = data->size; + + /* Now check the size received is the exact size expected */ + _exact_size = _max_size - (_state->io.segment_size - data->size); + if (VS_MBUF_SIZE(_mbuf) != _exact_size) + return -EBADMSG; + + return 0; +} + +EXPORT_SYMBOL(vs_client_block_io_getbufs_ack_read); +int vs_client_block_io_free_ack_read(struct vs_client_block_state *_state, + struct vs_pbuf *data, + struct vs_mbuf *_mbuf) +{ + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + + return 0; +} + +EXPORT_SYMBOL(vs_client_block_io_free_ack_read); +struct vs_mbuf *vs_client_block_io_alloc_req_write(struct vs_client_block_state + *_state, + struct vs_pbuf *data, + gfp_t flags) +{ + struct vs_mbuf *_mbuf; + const vs_message_id_t _msg_id = VSERVICE_BLOCK_IO_REQ_WRITE; + const uint32_t _msg_size = + sizeof(vs_message_id_t) + _state->io.segment_size + 32UL; + _mbuf = + vs_service_alloc_mbuf(VS_STATE_SERVICE_PTR(_state), _msg_size, + flags); + if (IS_ERR(_mbuf)) + return _mbuf; + if (!_mbuf) { + + WARN_ON_ONCE(1); + return ERR_PTR(-ENOMEM); + } + *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) = _msg_id; + + if (!data) + goto fail; + data->data = + (uintptr_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + + 28UL + sizeof(uint32_t)); + data->size = _state->io.segment_size; + data->max_size = data->size; + return _mbuf; + + fail: + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + return NULL; +} + +EXPORT_SYMBOL(vs_client_block_io_alloc_req_write); +int vs_client_block_io_free_req_write(struct vs_client_block_state *_state, + struct vs_pbuf *data, + struct vs_mbuf *_mbuf) +{ + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + + return 0; +} + +EXPORT_SYMBOL(vs_client_block_io_free_req_write); +int +vs_client_block_io_req_read(struct vs_client_block_state *_state, void *_opaque, + uint64_t sector_index, uint32_t num_sects, + bool nodelay, bool flush, gfp_t flags) +{ + struct vs_mbuf *_mbuf; + + const size_t _msg_size = sizeof(vs_message_id_t) + 24UL; + + struct vs_service_driver *vsdrv = + to_vs_service_driver(VS_STATE_SERVICE_PTR(_state)->dev.driver); + __maybe_unused struct vs_client_block *_client = + to_client_driver(vsdrv)->client; + uint32_t _opaque_tmp; + if (_state->state.base.statenum != VSERVICE_BASE_STATE_RUNNING) + return -EPROTO; + _opaque_tmp = + find_first_zero_bit(_state->state.io.read_bitmask, + VSERVICE_BLOCK_IO_READ_MAX_PENDING); + if (_opaque_tmp >= VSERVICE_BLOCK_IO_READ_MAX_PENDING) + return -EPROTO; + + _mbuf = + vs_service_alloc_mbuf(VS_STATE_SERVICE_PTR(_state), _msg_size, + flags); + if (IS_ERR(_mbuf)) + return PTR_ERR(_mbuf); + if (!_mbuf) { + + WARN_ON_ONCE(1); + + return -ENOMEM; + } + + *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) = VSERVICE_BLOCK_IO_REQ_READ; + + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 0UL) = + _opaque_tmp; + *(uint64_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 4UL) = + sector_index; + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 12UL) = + num_sects; + *(bool *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 16UL) = + nodelay; + *(bool *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 20UL) = + flush; + + { + int err = vs_service_send(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (err) { + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: Error %d sending message on transport.\n", + __func__, __LINE__, err); + + return err; + } + } + + _state->state.io.read_tags[_opaque_tmp] = _opaque; + __set_bit(_opaque_tmp, _state->state.io.read_bitmask); + + return 0; +} + +EXPORT_SYMBOL(vs_client_block_io_req_read); +int +vs_client_block_io_req_write(struct vs_client_block_state *_state, + void *_opaque, uint64_t sector_index, + uint32_t num_sects, bool nodelay, bool flush, + bool commit, struct vs_pbuf data, + struct vs_mbuf *_mbuf) +{ + struct vs_service_driver *vsdrv = + to_vs_service_driver(VS_STATE_SERVICE_PTR(_state)->dev.driver); + __maybe_unused struct vs_client_block *_client = + to_client_driver(vsdrv)->client; + uint32_t _opaque_tmp; + if (_state->state.base.statenum != VSERVICE_BASE_STATE_RUNNING) + return -EPROTO; + _opaque_tmp = + find_first_zero_bit(_state->state.io.write_bitmask, + VSERVICE_BLOCK_IO_WRITE_MAX_PENDING); + if (_opaque_tmp >= VSERVICE_BLOCK_IO_WRITE_MAX_PENDING) + return -EPROTO; + + if (*(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) != + VSERVICE_BLOCK_IO_REQ_WRITE) + + return -EINVAL; + + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 0UL) = + _opaque_tmp; + *(uint64_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 4UL) = + sector_index; + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 12UL) = + num_sects; + *(bool *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 16UL) = + nodelay; + *(bool *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 20UL) = + flush; + *(bool *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 24UL) = + commit; + if ((data.size + sizeof(vs_message_id_t) + 28UL) > VS_MBUF_SIZE(_mbuf)) + return -EINVAL; + + if (data.size < data.max_size) + VS_MBUF_SIZE(_mbuf) -= (data.max_size - data.size); + + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 28UL) = + data.size; + + { + int err = vs_service_send(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (err) { + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: Error %d sending message on transport.\n", + __func__, __LINE__, err); + + return err; + } + } + + _state->state.io.write_tags[_opaque_tmp] = _opaque; + __set_bit(_opaque_tmp, _state->state.io.write_bitmask); + + return 0; +} + +EXPORT_SYMBOL(vs_client_block_io_req_write); +static int +block_io_handle_ack_read(const struct vs_client_block *_client, + struct vs_client_block_state *_state, + struct vs_mbuf *_mbuf) +{ + const size_t _max_size = + sizeof(vs_message_id_t) + _state->io.segment_size + 8UL; + void *_opaque; + struct vs_pbuf data; + const size_t _min_size = _max_size - _state->io.segment_size; + size_t _exact_size; + uint32_t _opaque_tmp; + + /* The first check is to ensure the message isn't complete garbage */ + if ((VS_MBUF_SIZE(_mbuf) > _max_size) + || (VS_MBUF_SIZE(_mbuf) < _min_size)) + return -EBADMSG; + _opaque_tmp = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 0UL); + if (_opaque_tmp >= VSERVICE_BLOCK_IO_READ_MAX_PENDING) + return -EPROTO; + if (!VSERVICE_BASE_STATE_IS_RUNNING(_state->state.base)) + return -EPROTO; + if (!test_bit(_opaque_tmp, _state->state.io.read_bitmask)) + return -EPROTO; + _opaque = _state->state.io.read_tags[_opaque_tmp]; + __clear_bit(_opaque_tmp, _state->state.io.read_bitmask); + + data.size = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 4UL); + data.data = + (uintptr_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 4UL + + sizeof(uint32_t)); + data.max_size = data.size; + + /* Now check the size received is the exact size expected */ + _exact_size = _max_size - (_state->io.segment_size - data.size); + if (VS_MBUF_SIZE(_mbuf) != _exact_size) + return -EBADMSG; + if (_client->io.ack_read) + return _client->io.ack_read(_state, _opaque, data, _mbuf); + return 0; +} + +static int +block_io_handle_nack_read(const struct vs_client_block *_client, + struct vs_client_block_state *_state, + struct vs_mbuf *_mbuf) +{ + const size_t _expected_size = sizeof(vs_message_id_t) + 8UL; + void *_opaque; + vservice_block_block_io_error_t err; + uint32_t _opaque_tmp; + + if (VS_MBUF_SIZE(_mbuf) < _expected_size) + return -EBADMSG; + + _opaque_tmp = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 0UL); + if (_opaque_tmp >= VSERVICE_BLOCK_IO_READ_MAX_PENDING) + return -EPROTO; + if (!VSERVICE_BASE_STATE_IS_RUNNING(_state->state.base)) + return -EPROTO; + if (!test_bit(_opaque_tmp, _state->state.io.read_bitmask)) + return -EPROTO; + _opaque = _state->state.io.read_tags[_opaque_tmp]; + __clear_bit(_opaque_tmp, _state->state.io.read_bitmask); + err = + *(vservice_block_block_io_error_t *) (VS_MBUF_DATA(_mbuf) + + sizeof(vs_message_id_t) + + 4UL); + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (_client->io.nack_read) + return _client->io.nack_read(_state, _opaque, err); + return 0; +} + +EXPORT_SYMBOL(block_io_handle_ack_read); +static int +block_io_handle_ack_write(const struct vs_client_block *_client, + struct vs_client_block_state *_state, + struct vs_mbuf *_mbuf) +{ + const size_t _expected_size = sizeof(vs_message_id_t) + 4UL; + void *_opaque; + uint32_t _opaque_tmp; + + if (VS_MBUF_SIZE(_mbuf) < _expected_size) + return -EBADMSG; + + _opaque_tmp = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 0UL); + if (_opaque_tmp >= VSERVICE_BLOCK_IO_WRITE_MAX_PENDING) + return -EPROTO; + if (!VSERVICE_BASE_STATE_IS_RUNNING(_state->state.base)) + return -EPROTO; + if (!test_bit(_opaque_tmp, _state->state.io.write_bitmask)) + return -EPROTO; + _opaque = _state->state.io.write_tags[_opaque_tmp]; + __clear_bit(_opaque_tmp, _state->state.io.write_bitmask); + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (_client->io.ack_write) + return _client->io.ack_write(_state, _opaque); + return 0; +} + +static int +block_io_handle_nack_write(const struct vs_client_block *_client, + struct vs_client_block_state *_state, + struct vs_mbuf *_mbuf) +{ + const size_t _expected_size = sizeof(vs_message_id_t) + 8UL; + void *_opaque; + vservice_block_block_io_error_t err; + uint32_t _opaque_tmp; + + if (VS_MBUF_SIZE(_mbuf) < _expected_size) + return -EBADMSG; + + _opaque_tmp = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 0UL); + if (_opaque_tmp >= VSERVICE_BLOCK_IO_WRITE_MAX_PENDING) + return -EPROTO; + if (!VSERVICE_BASE_STATE_IS_RUNNING(_state->state.base)) + return -EPROTO; + if (!test_bit(_opaque_tmp, _state->state.io.write_bitmask)) + return -EPROTO; + _opaque = _state->state.io.write_tags[_opaque_tmp]; + __clear_bit(_opaque_tmp, _state->state.io.write_bitmask); + err = + *(vservice_block_block_io_error_t *) (VS_MBUF_DATA(_mbuf) + + sizeof(vs_message_id_t) + + 4UL); + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (_client->io.nack_write) + return _client->io.nack_write(_state, _opaque, err); + return 0; +} + +EXPORT_SYMBOL(block_io_handle_ack_write); +static int +block_handle_message(struct vs_service_device *service, struct vs_mbuf *_mbuf) +{ + vs_message_id_t message_id; + __maybe_unused struct vs_client_block_state *state = + dev_get_drvdata(&service->dev); + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + __maybe_unused struct vs_client_block *client = + to_client_driver(vsdrv)->client; + + int ret; + + /* Extract the message ID */ + if (VS_MBUF_SIZE(_mbuf) < sizeof(message_id)) { + dev_err(&state->service->dev, + "[%s:%d] Protocol error: Invalid message size %zd\n", + __func__, __LINE__, VS_MBUF_SIZE(_mbuf)); + + return -EBADMSG; + } + + message_id = *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)); + + switch (message_id) { + +/** interface base **/ +/* command in sync open */ + case VSERVICE_BLOCK_BASE_ACK_OPEN: + ret = block_base_handle_ack_open(client, state, _mbuf); + break; + case VSERVICE_BLOCK_BASE_NACK_OPEN: + ret = block_base_handle_nack_open(client, state, _mbuf); + break; + +/* command in sync close */ + case VSERVICE_BLOCK_BASE_ACK_CLOSE: + ret = block_base_handle_ack_close(client, state, _mbuf); + break; + case VSERVICE_BLOCK_BASE_NACK_CLOSE: + ret = block_base_handle_nack_close(client, state, _mbuf); + break; + +/* command in sync reopen */ + case VSERVICE_BLOCK_BASE_ACK_REOPEN: + ret = block_base_handle_ack_reopen(client, state, _mbuf); + break; + case VSERVICE_BLOCK_BASE_NACK_REOPEN: + ret = block_base_handle_nack_reopen(client, state, _mbuf); + break; + +/** interface block_io **/ +/* command in parallel read */ + case VSERVICE_BLOCK_IO_ACK_READ: + ret = block_io_handle_ack_read(client, state, _mbuf); + break; + case VSERVICE_BLOCK_IO_NACK_READ: + ret = block_io_handle_nack_read(client, state, _mbuf); + break; + +/* command in parallel write */ + case VSERVICE_BLOCK_IO_ACK_WRITE: + ret = block_io_handle_ack_write(client, state, _mbuf); + break; + case VSERVICE_BLOCK_IO_NACK_WRITE: + ret = block_io_handle_nack_write(client, state, _mbuf); + break; + + default: + dev_err(&state->service->dev, + "[%s:%d] Protocol error: Unknown message type %d\n", + __func__, __LINE__, (int)message_id); + + ret = -EPROTO; + break; + } + + if (ret) { + dev_err(&state->service->dev, + "[%s:%d] Protocol error: Handler for message type %d returned %d\n", + __func__, __LINE__, (int)message_id, ret); + + } + + return ret; +} + +static void block_handle_notify(struct vs_service_device *service, + uint32_t notify_bits) +{ + __maybe_unused struct vs_client_block_state *state = + dev_get_drvdata(&service->dev); + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + __maybe_unused struct vs_client_block *client = + to_client_driver(vsdrv)->client; + + uint32_t bits = notify_bits; + int ret; + + while (bits) { + uint32_t not = __ffs(bits); + switch (not) { + + /** interface block_io **/ + + default: + dev_err(&state->service->dev, + "[%s:%d] Protocol error: Unknown notification %d\n", + __func__, __LINE__, (int)not); + + ret = -EPROTO; + break; + + } + bits &= ~(1 << not); + if (ret) { + dev_err(&state->service->dev, + "[%s:%d] Protocol error: Handler for notification %d returned %d\n", + __func__, __LINE__, (int)not, ret); + + } + } +} + +int vs_client_block_reopen(struct vs_client_block_state *_state) +{ + return _vs_client_block_req_reopen(_state); +} + +EXPORT_SYMBOL(vs_client_block_reopen); + +int vs_client_block_close(struct vs_client_block_state *_state) +{ + return _vs_client_block_req_close(_state); +} + +EXPORT_SYMBOL(vs_client_block_close); + +MODULE_DESCRIPTION("OKL4 Virtual Services blockClient Protocol Driver"); +MODULE_AUTHOR("Open Kernel Labs, Inc"); diff --git a/drivers/vservices/protocol/block/server.c b/drivers/vservices/protocol/block/server.c new file mode 100644 index 000000000000..a4a7d1a0c214 --- /dev/null +++ b/drivers/vservices/protocol/block/server.c @@ -0,0 +1,1371 @@ + +/* + * Copyright (c) 2012-2018 General Dynamics + * Copyright (c) 2014 Open Kernel Labs, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + + /* + * This is the generated code for the block server protocol handling. + */ +#include +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0) +#include +#endif + +#include +#include +#include +#include +#include +#include + +#include "../../transport.h" + +#define VS_MBUF_SIZE(mbuf) mbuf->size +#define VS_MBUF_DATA(mbuf) mbuf->data +#define VS_STATE_SERVICE_PTR(state) state->service + +/*** Linux driver model integration ***/ +struct vs_block_server_driver { + struct vs_server_block *server; + struct list_head list; + struct vs_service_driver vsdrv; +}; + +#define to_server_driver(d) \ + container_of(d, struct vs_block_server_driver, vsdrv) + +static void reset_nack_requests(struct vs_service_device *service) +{ + +} + +static void block_handle_start(struct vs_service_device *service) +{ + + struct vs_server_block_state *state = dev_get_drvdata(&service->dev); + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + struct vs_server_block *server __maybe_unused = + to_server_driver(vsdrv)->server; + + vs_service_state_lock(service); + state->state = VSERVICE_BLOCK_RESET_STATE; + + vs_service_state_unlock(service); +} + +static void block_handle_reset(struct vs_service_device *service) +{ + + struct vs_server_block_state *state = dev_get_drvdata(&service->dev); + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + struct vs_server_block *server __maybe_unused = + to_server_driver(vsdrv)->server; + + vs_service_state_lock(service); + if (!VSERVICE_BASE_STATE_IS_RUNNING(state->state.base)) { + vs_service_state_unlock(service); + return; + } + state->state.base = VSERVICE_BASE_RESET_STATE; + reset_nack_requests(service); + if (server->closed) + server->closed(state); + + state->state = VSERVICE_BLOCK_RESET_STATE; + + vs_service_state_unlock(service); +} + +static void block_handle_start_bh(struct vs_service_device *service) +{ + + struct vs_server_block_state *state = dev_get_drvdata(&service->dev); + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + struct vs_server_block *server __maybe_unused = + to_server_driver(vsdrv)->server; + + vs_service_state_lock_bh(service); + state->state = VSERVICE_BLOCK_RESET_STATE; + + vs_service_state_unlock_bh(service); +} + +static void block_handle_reset_bh(struct vs_service_device *service) +{ + + struct vs_server_block_state *state = dev_get_drvdata(&service->dev); + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + struct vs_server_block *server __maybe_unused = + to_server_driver(vsdrv)->server; + + vs_service_state_lock_bh(service); + if (!VSERVICE_BASE_STATE_IS_RUNNING(state->state.base)) { + vs_service_state_unlock_bh(service); + return; + } + state->state.base = VSERVICE_BASE_RESET_STATE; + reset_nack_requests(service); + if (server->closed) + server->closed(state); + + state->state = VSERVICE_BLOCK_RESET_STATE; + + vs_service_state_unlock_bh(service); +} + +static int block_server_probe(struct vs_service_device *service); +static int block_server_remove(struct vs_service_device *service); +static int block_handle_message(struct vs_service_device *service, + struct vs_mbuf *_mbuf); +static void block_handle_notify(struct vs_service_device *service, + uint32_t flags); +static void block_handle_start(struct vs_service_device *service); +static void block_handle_start_bh(struct vs_service_device *service); +static void block_handle_reset(struct vs_service_device *service); +static void block_handle_reset_bh(struct vs_service_device *service); +static int block_handle_tx_ready(struct vs_service_device *service); + +int __vservice_block_server_register(struct vs_server_block *server, + const char *name, struct module *owner) +{ + int ret; + struct vs_block_server_driver *driver; + + if (server->tx_atomic && !server->rx_atomic) + return -EINVAL; + + driver = kzalloc(sizeof(*driver), GFP_KERNEL); + if (!driver) { + ret = -ENOMEM; + goto fail_alloc_driver; + } + + server->driver = &driver->vsdrv; + driver->server = server; + + driver->vsdrv.protocol = VSERVICE_BLOCK_PROTOCOL_NAME; + + driver->vsdrv.is_server = true; + driver->vsdrv.rx_atomic = server->rx_atomic; + driver->vsdrv.tx_atomic = server->tx_atomic; + /* FIXME Jira ticket SDK-2835 - philipd. */ + driver->vsdrv.in_quota_min = 1; + driver->vsdrv.in_quota_best = server->in_quota_best ? + server->in_quota_best : driver->vsdrv.in_quota_min; + /* FIXME Jira ticket SDK-2835 - philipd. */ + driver->vsdrv.out_quota_min = 1; + driver->vsdrv.out_quota_best = server->out_quota_best ? + server->out_quota_best : driver->vsdrv.out_quota_min; + driver->vsdrv.in_notify_count = VSERVICE_BLOCK_NBIT_IN__COUNT; + driver->vsdrv.out_notify_count = VSERVICE_BLOCK_NBIT_OUT__COUNT; + + driver->vsdrv.probe = block_server_probe; + driver->vsdrv.remove = block_server_remove; + driver->vsdrv.receive = block_handle_message; + driver->vsdrv.notify = block_handle_notify; + driver->vsdrv.start = server->tx_atomic ? + block_handle_start_bh : block_handle_start; + driver->vsdrv.reset = server->tx_atomic ? + block_handle_reset_bh : block_handle_reset; + driver->vsdrv.tx_ready = block_handle_tx_ready; + driver->vsdrv.out_notify_count = 0; + driver->vsdrv.in_notify_count = 0; + driver->vsdrv.driver.name = name; + driver->vsdrv.driver.owner = owner; + driver->vsdrv.driver.bus = &vs_server_bus_type; + + ret = driver_register(&driver->vsdrv.driver); + + if (ret) { + goto fail_driver_register; + } + + return 0; + + fail_driver_register: + server->driver = NULL; + kfree(driver); + fail_alloc_driver: + return ret; +} + +EXPORT_SYMBOL(__vservice_block_server_register); + +int vservice_block_server_unregister(struct vs_server_block *server) +{ + struct vs_block_server_driver *driver; + + if (!server->driver) + return 0; + + driver = to_server_driver(server->driver); + driver_unregister(&driver->vsdrv.driver); + + server->driver = NULL; + kfree(driver); + + return 0; +} + +EXPORT_SYMBOL(vservice_block_server_unregister); + +static int block_server_probe(struct vs_service_device *service) +{ + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + struct vs_server_block *server = to_server_driver(vsdrv)->server; + struct vs_server_block_state *state; + + state = server->alloc(service); + if (!state) + return -ENOMEM; + else if (IS_ERR(state)) + return PTR_ERR(state); + + state->service = vs_get_service(service); + state->state = VSERVICE_BLOCK_RESET_STATE; + + dev_set_drvdata(&service->dev, state); + + return 0; +} + +static int block_server_remove(struct vs_service_device *service) +{ + struct vs_server_block_state *state = dev_get_drvdata(&service->dev); + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + struct vs_server_block *server = to_server_driver(vsdrv)->server; + + state->released = true; + dev_set_drvdata(&service->dev, NULL); + server->release(state); + + vs_put_service(service); + + return 0; +} + +static int block_handle_tx_ready(struct vs_service_device *service) +{ + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + struct vs_server_block *server = to_server_driver(vsdrv)->server; + struct vs_server_block_state *state = dev_get_drvdata(&service->dev); + + if (!VSERVICE_BASE_STATE_IS_RUNNING(state->state.base)) + return 0; + + if (server->tx_ready) + server->tx_ready(state); + + return 0; +} + +static int +vs_server_block_send_ack_open(struct vs_server_block_state *_state, gfp_t flags) +{ + struct vs_mbuf *_mbuf; + + const size_t _msg_size = sizeof(vs_message_id_t) + 28UL; + + struct vs_service_driver *vsdrv = + to_vs_service_driver(VS_STATE_SERVICE_PTR(_state)->dev.driver); + __maybe_unused struct vs_server_block *_server = + to_server_driver(vsdrv)->server; + + _mbuf = + vs_service_alloc_mbuf(VS_STATE_SERVICE_PTR(_state), _msg_size, + flags); + if (IS_ERR(_mbuf)) + return PTR_ERR(_mbuf); + if (!_mbuf) { + + WARN_ON_ONCE(1); + + return -ENOMEM; + } + + *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) = + VSERVICE_BLOCK_BASE_ACK_OPEN; + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_CLOSED__OPEN: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + *(bool *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 0UL) = + _state->readonly; + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 4UL) = + _state->sector_size; + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 8UL) = + _state->segment_size; + *(uint64_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 12UL) = + _state->device_sectors; + *(bool *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 20UL) = + _state->flushable; + *(bool *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 24UL) = + _state->committable; + _state->io.sector_size = _state->sector_size; + _state->io.segment_size = _state->segment_size; + + { + int err = vs_service_send(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (err) { + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: Error %d sending message on transport.\n", + __func__, __LINE__, err); + + return err; + } + } + + _state->state.base.statenum = VSERVICE_BASE_STATE_RUNNING; + + return 0; +} + +EXPORT_SYMBOL(vs_server_block_send_ack_open); +static int +vs_server_block_send_nack_open(struct vs_server_block_state *_state, + gfp_t flags) +{ + struct vs_mbuf *_mbuf; + + const size_t _msg_size = sizeof(vs_message_id_t) + 0UL; + + struct vs_service_driver *vsdrv = + to_vs_service_driver(VS_STATE_SERVICE_PTR(_state)->dev.driver); + __maybe_unused struct vs_server_block *_server = + to_server_driver(vsdrv)->server; + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_CLOSED__OPEN: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + + _mbuf = + vs_service_alloc_mbuf(VS_STATE_SERVICE_PTR(_state), _msg_size, + flags); + if (IS_ERR(_mbuf)) + return PTR_ERR(_mbuf); + if (!_mbuf) { + + WARN_ON_ONCE(1); + + return -ENOMEM; + } + + *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) = + VSERVICE_BLOCK_BASE_NACK_OPEN; + + { + int err = vs_service_send(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (err) { + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: Error %d sending message on transport.\n", + __func__, __LINE__, err); + + return err; + } + } + + _state->state.base.statenum = VSERVICE_BASE_STATE_CLOSED; + + return 0; +} + +EXPORT_SYMBOL(vs_server_block_send_nack_open); +static int +vs_server_block_send_ack_close(struct vs_server_block_state *_state, + gfp_t flags) +{ + struct vs_mbuf *_mbuf; + + const size_t _msg_size = sizeof(vs_message_id_t) + 0UL; + + struct vs_service_driver *vsdrv = + to_vs_service_driver(VS_STATE_SERVICE_PTR(_state)->dev.driver); + __maybe_unused struct vs_server_block *_server = + to_server_driver(vsdrv)->server; + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_RUNNING__CLOSE: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + + _mbuf = + vs_service_alloc_mbuf(VS_STATE_SERVICE_PTR(_state), _msg_size, + flags); + if (IS_ERR(_mbuf)) + return PTR_ERR(_mbuf); + if (!_mbuf) { + + WARN_ON_ONCE(1); + + return -ENOMEM; + } + + *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) = + VSERVICE_BLOCK_BASE_ACK_CLOSE; + + { + int err = vs_service_send(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (err) { + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: Error %d sending message on transport.\n", + __func__, __LINE__, err); + + return err; + } + } + + _state->state.base.statenum = VSERVICE_BASE_STATE_CLOSED; + + return 0; +} + +EXPORT_SYMBOL(vs_server_block_send_ack_close); +static int +vs_server_block_send_nack_close(struct vs_server_block_state *_state, + gfp_t flags) +{ + struct vs_mbuf *_mbuf; + + const size_t _msg_size = sizeof(vs_message_id_t) + 0UL; + + struct vs_service_driver *vsdrv = + to_vs_service_driver(VS_STATE_SERVICE_PTR(_state)->dev.driver); + __maybe_unused struct vs_server_block *_server = + to_server_driver(vsdrv)->server; + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_RUNNING__CLOSE: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + + _mbuf = + vs_service_alloc_mbuf(VS_STATE_SERVICE_PTR(_state), _msg_size, + flags); + if (IS_ERR(_mbuf)) + return PTR_ERR(_mbuf); + if (!_mbuf) { + + WARN_ON_ONCE(1); + + return -ENOMEM; + } + + *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) = + VSERVICE_BLOCK_BASE_NACK_CLOSE; + + { + int err = vs_service_send(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (err) { + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: Error %d sending message on transport.\n", + __func__, __LINE__, err); + + return err; + } + } + + _state->state.base.statenum = VSERVICE_BASE_STATE_RUNNING; + + return 0; +} + +EXPORT_SYMBOL(vs_server_block_send_nack_close); +static int +vs_server_block_send_ack_reopen(struct vs_server_block_state *_state, + gfp_t flags) +{ + struct vs_mbuf *_mbuf; + + const size_t _msg_size = sizeof(vs_message_id_t) + 0UL; + + struct vs_service_driver *vsdrv = + to_vs_service_driver(VS_STATE_SERVICE_PTR(_state)->dev.driver); + __maybe_unused struct vs_server_block *_server = + to_server_driver(vsdrv)->server; + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_RUNNING__REOPEN: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + + _mbuf = + vs_service_alloc_mbuf(VS_STATE_SERVICE_PTR(_state), _msg_size, + flags); + if (IS_ERR(_mbuf)) + return PTR_ERR(_mbuf); + if (!_mbuf) { + + WARN_ON_ONCE(1); + + return -ENOMEM; + } + + *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) = + VSERVICE_BLOCK_BASE_ACK_REOPEN; + + { + int err = vs_service_send(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (err) { + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: Error %d sending message on transport.\n", + __func__, __LINE__, err); + + return err; + } + } + + _state->state.base.statenum = VSERVICE_BASE__RESET; + + return 0; +} + +EXPORT_SYMBOL(vs_server_block_send_ack_reopen); +static int +vs_server_block_send_nack_reopen(struct vs_server_block_state *_state, + gfp_t flags) +{ + struct vs_mbuf *_mbuf; + + const size_t _msg_size = sizeof(vs_message_id_t) + 0UL; + + struct vs_service_driver *vsdrv = + to_vs_service_driver(VS_STATE_SERVICE_PTR(_state)->dev.driver); + __maybe_unused struct vs_server_block *_server = + to_server_driver(vsdrv)->server; + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_RUNNING__REOPEN: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + + _mbuf = + vs_service_alloc_mbuf(VS_STATE_SERVICE_PTR(_state), _msg_size, + flags); + if (IS_ERR(_mbuf)) + return PTR_ERR(_mbuf); + if (!_mbuf) { + + WARN_ON_ONCE(1); + + return -ENOMEM; + } + + *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) = + VSERVICE_BLOCK_BASE_NACK_REOPEN; + + { + int err = vs_service_send(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (err) { + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: Error %d sending message on transport.\n", + __func__, __LINE__, err); + + return err; + } + } + + _state->state.base.statenum = VSERVICE_BASE_STATE_RUNNING; + + return 0; +} + +EXPORT_SYMBOL(vs_server_block_send_nack_reopen); +static int +vs_server_block_handle_req_open(const struct vs_server_block *_server, + struct vs_server_block_state *_state, + struct vs_mbuf *_mbuf) +{ + const size_t _expected_size = sizeof(vs_message_id_t) + 0UL; + + if (VS_MBUF_SIZE(_mbuf) < _expected_size) + return -EBADMSG; + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_CLOSED: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + _state->state.base.statenum = VSERVICE_BASE_STATE_CLOSED__OPEN; + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (_server->open) + return vs_server_block_open_complete(_state, + _server->open(_state)); + return vs_server_block_open_complete(_state, VS_SERVER_RESP_SUCCESS); + +} + +int vs_server_block_open_complete(struct vs_server_block_state *_state, + vs_server_response_type_t resp) +{ + int ret = 0; + if (resp == VS_SERVER_RESP_SUCCESS) + ret = + vs_server_block_send_ack_open(_state, + vs_service_has_atomic_rx + (VS_STATE_SERVICE_PTR(_state)) + ? GFP_ATOMIC : GFP_KERNEL); + else if (resp == VS_SERVER_RESP_FAILURE) + ret = + vs_server_block_send_nack_open(_state, + vs_service_has_atomic_rx + (VS_STATE_SERVICE_PTR + (_state)) ? GFP_ATOMIC : + GFP_KERNEL); + + return ret; + +} + +EXPORT_SYMBOL(vs_server_block_open_complete); + +EXPORT_SYMBOL(vs_server_block_handle_req_open); +static int +vs_server_block_handle_req_close(const struct vs_server_block *_server, + struct vs_server_block_state *_state, + struct vs_mbuf *_mbuf) +{ + const size_t _expected_size = sizeof(vs_message_id_t) + 0UL; + + if (VS_MBUF_SIZE(_mbuf) < _expected_size) + return -EBADMSG; + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_RUNNING: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + _state->state.base.statenum = VSERVICE_BASE_STATE_RUNNING__CLOSE; + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (_server->close) + return vs_server_block_close_complete(_state, + _server->close(_state)); + return vs_server_block_close_complete(_state, VS_SERVER_RESP_SUCCESS); + +} + +int vs_server_block_close_complete(struct vs_server_block_state *_state, + vs_server_response_type_t resp) +{ + int ret = 0; + if (resp == VS_SERVER_RESP_SUCCESS) + ret = + vs_server_block_send_ack_close(_state, + vs_service_has_atomic_rx + (VS_STATE_SERVICE_PTR + (_state)) ? GFP_ATOMIC : + GFP_KERNEL); + else if (resp == VS_SERVER_RESP_FAILURE) + ret = + vs_server_block_send_nack_close(_state, + vs_service_has_atomic_rx + (VS_STATE_SERVICE_PTR + (_state)) ? GFP_ATOMIC : + GFP_KERNEL); + if ((resp == VS_SERVER_RESP_SUCCESS) && (ret == 0)) { + wake_up_all(&_state->service->quota_wq); + } + return ret; + +} + +EXPORT_SYMBOL(vs_server_block_close_complete); + +EXPORT_SYMBOL(vs_server_block_handle_req_close); +static int +vs_server_block_handle_req_reopen(const struct vs_server_block *_server, + struct vs_server_block_state *_state, + struct vs_mbuf *_mbuf) +{ + const size_t _expected_size = sizeof(vs_message_id_t) + 0UL; + + if (VS_MBUF_SIZE(_mbuf) < _expected_size) + return -EBADMSG; + + switch (_state->state.base.statenum) { + case VSERVICE_BASE_STATE_RUNNING: + + break; + + default: + dev_err(&_state->service->dev, + "[%s:%d] Protocol error: In wrong protocol state %d - %s\n", + __func__, __LINE__, _state->state.base.statenum, + vservice_base_get_state_string(_state->state.base)); + + return -EPROTO; + + } + _state->state.base.statenum = VSERVICE_BASE_STATE_RUNNING__REOPEN; + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (_server->reopen) + return vs_server_block_reopen_complete(_state, + _server->reopen(_state)); + else + return vs_server_block_send_nack_reopen(_state, + vs_service_has_atomic_rx + (VS_STATE_SERVICE_PTR + (_state)) ? GFP_ATOMIC + : GFP_KERNEL); + +} + +int vs_server_block_reopen_complete(struct vs_server_block_state *_state, + vs_server_response_type_t resp) +{ + int ret = 0; + if (resp == VS_SERVER_RESP_SUCCESS) { + _state->io.sector_size = _state->sector_size; + _state->io.segment_size = _state->segment_size; + ret = + vs_server_block_send_ack_reopen(_state, + vs_service_has_atomic_rx + (VS_STATE_SERVICE_PTR + (_state)) ? GFP_ATOMIC : + GFP_KERNEL); + } else if (resp == VS_SERVER_RESP_FAILURE) { + ret = + vs_server_block_send_nack_reopen(_state, + vs_service_has_atomic_rx + (VS_STATE_SERVICE_PTR + (_state)) ? GFP_ATOMIC : + GFP_KERNEL); + } + + return ret; + +} + +EXPORT_SYMBOL(vs_server_block_reopen_complete); + +EXPORT_SYMBOL(vs_server_block_handle_req_reopen); +struct vs_mbuf *vs_server_block_io_alloc_ack_read(struct vs_server_block_state + *_state, struct vs_pbuf *data, + gfp_t flags) +{ + struct vs_mbuf *_mbuf; + const vs_message_id_t _msg_id = VSERVICE_BLOCK_IO_ACK_READ; + const uint32_t _msg_size = + sizeof(vs_message_id_t) + _state->io.segment_size + 8UL; + _mbuf = + vs_service_alloc_mbuf(VS_STATE_SERVICE_PTR(_state), _msg_size, + flags); + if (IS_ERR(_mbuf)) + return _mbuf; + if (!_mbuf) { + + WARN_ON_ONCE(1); + return ERR_PTR(-ENOMEM); + } + *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) = _msg_id; + + if (!data) + goto fail; + data->data = + (uintptr_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 4UL + + sizeof(uint32_t)); + data->size = _state->io.segment_size; + data->max_size = data->size; + return _mbuf; + + fail: + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + return NULL; +} + +EXPORT_SYMBOL(vs_server_block_io_alloc_ack_read); +int vs_server_block_io_free_ack_read(struct vs_server_block_state *_state, + struct vs_pbuf *data, + struct vs_mbuf *_mbuf) +{ + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + + return 0; +} + +EXPORT_SYMBOL(vs_server_block_io_free_ack_read); +int vs_server_block_io_getbufs_req_write(struct vs_server_block_state *_state, + struct vs_pbuf *data, + struct vs_mbuf *_mbuf) +{ + const vs_message_id_t _msg_id = VSERVICE_BLOCK_IO_REQ_WRITE; + const size_t _max_size = + sizeof(vs_message_id_t) + _state->io.segment_size + 32UL; + const size_t _min_size = _max_size - _state->io.segment_size; + size_t _exact_size; + + if (*(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) != _msg_id) + return -EINVAL; + if ((VS_MBUF_SIZE(_mbuf) > _max_size) + || (VS_MBUF_SIZE(_mbuf) < _min_size)) + return -EBADMSG; + + data->size = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + + 28UL); + data->data = + (uintptr_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + + 28UL + sizeof(uint32_t)); + data->max_size = data->size; + + /* Now check the size received is the exact size expected */ + _exact_size = _max_size - (_state->io.segment_size - data->size); + if (VS_MBUF_SIZE(_mbuf) != _exact_size) + return -EBADMSG; + + return 0; +} + +EXPORT_SYMBOL(vs_server_block_io_getbufs_req_write); +int vs_server_block_io_free_req_write(struct vs_server_block_state *_state, + struct vs_pbuf *data, + struct vs_mbuf *_mbuf) +{ + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + + return 0; +} + +EXPORT_SYMBOL(vs_server_block_io_free_req_write); +int +vs_server_block_io_send_ack_read(struct vs_server_block_state *_state, + uint32_t _opaque, struct vs_pbuf data, + struct vs_mbuf *_mbuf) +{ + struct vs_service_driver *vsdrv = + to_vs_service_driver(VS_STATE_SERVICE_PTR(_state)->dev.driver); + __maybe_unused struct vs_server_block *_server = + to_server_driver(vsdrv)->server; + + if (_opaque >= VSERVICE_BLOCK_IO_READ_MAX_PENDING) + return -EPROTO; + if (!VSERVICE_BASE_STATE_IS_RUNNING(_state->state.base)) + return -EPROTO; + if (!test_bit(_opaque, _state->state.io.read_bitmask)) + return -EPROTO; + if (*(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) != + VSERVICE_BLOCK_IO_ACK_READ) + + return -EINVAL; + + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 0UL) = + _opaque; + if ((data.size + sizeof(vs_message_id_t) + 4UL) > VS_MBUF_SIZE(_mbuf)) + return -EINVAL; + + if (data.size < data.max_size) + VS_MBUF_SIZE(_mbuf) -= (data.max_size - data.size); + + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 4UL) = + data.size; + + { + int err = vs_service_send(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (err) { + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: Error %d sending message on transport.\n", + __func__, __LINE__, err); + + return err; + } + } + + __clear_bit(_opaque, _state->state.io.read_bitmask); + + return 0; +} + +EXPORT_SYMBOL(vs_server_block_io_send_ack_read); +int +vs_server_block_io_send_nack_read(struct vs_server_block_state *_state, + uint32_t _opaque, + vservice_block_block_io_error_t err, + gfp_t flags) +{ + struct vs_mbuf *_mbuf; + + const size_t _msg_size = sizeof(vs_message_id_t) + 8UL; + + struct vs_service_driver *vsdrv = + to_vs_service_driver(VS_STATE_SERVICE_PTR(_state)->dev.driver); + __maybe_unused struct vs_server_block *_server = + to_server_driver(vsdrv)->server; + + if (_opaque >= VSERVICE_BLOCK_IO_READ_MAX_PENDING) + return -EPROTO; + if (!VSERVICE_BASE_STATE_IS_RUNNING(_state->state.base)) + return -EPROTO; + if (!test_bit(_opaque, _state->state.io.read_bitmask)) + return -EPROTO; + + _mbuf = + vs_service_alloc_mbuf(VS_STATE_SERVICE_PTR(_state), _msg_size, + flags); + if (IS_ERR(_mbuf)) + return PTR_ERR(_mbuf); + if (!_mbuf) { + + WARN_ON_ONCE(1); + + return -ENOMEM; + } + + *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) = + VSERVICE_BLOCK_IO_NACK_READ; + + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 0UL) = + _opaque; + *(vservice_block_block_io_error_t *) (VS_MBUF_DATA(_mbuf) + + sizeof(vs_message_id_t) + 4UL) = + err; + + { + int err = vs_service_send(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (err) { + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: Error %d sending message on transport.\n", + __func__, __LINE__, err); + + return err; + } + } + + __clear_bit(_opaque, _state->state.io.read_bitmask); + + return 0; +} + +EXPORT_SYMBOL(vs_server_block_io_send_nack_read); +int +vs_server_block_io_send_ack_write(struct vs_server_block_state *_state, + uint32_t _opaque, gfp_t flags) +{ + struct vs_mbuf *_mbuf; + + const size_t _msg_size = sizeof(vs_message_id_t) + 4UL; + + struct vs_service_driver *vsdrv = + to_vs_service_driver(VS_STATE_SERVICE_PTR(_state)->dev.driver); + __maybe_unused struct vs_server_block *_server = + to_server_driver(vsdrv)->server; + + if (_opaque >= VSERVICE_BLOCK_IO_WRITE_MAX_PENDING) + return -EPROTO; + if (!VSERVICE_BASE_STATE_IS_RUNNING(_state->state.base)) + return -EPROTO; + if (!test_bit(_opaque, _state->state.io.write_bitmask)) + return -EPROTO; + + _mbuf = + vs_service_alloc_mbuf(VS_STATE_SERVICE_PTR(_state), _msg_size, + flags); + if (IS_ERR(_mbuf)) + return PTR_ERR(_mbuf); + if (!_mbuf) { + + WARN_ON_ONCE(1); + + return -ENOMEM; + } + + *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) = + VSERVICE_BLOCK_IO_ACK_WRITE; + + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 0UL) = + _opaque; + + { + int err = vs_service_send(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (err) { + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: Error %d sending message on transport.\n", + __func__, __LINE__, err); + + return err; + } + } + + __clear_bit(_opaque, _state->state.io.write_bitmask); + + return 0; +} + +EXPORT_SYMBOL(vs_server_block_io_send_ack_write); +int +vs_server_block_io_send_nack_write(struct vs_server_block_state *_state, + uint32_t _opaque, + vservice_block_block_io_error_t err, + gfp_t flags) +{ + struct vs_mbuf *_mbuf; + + const size_t _msg_size = sizeof(vs_message_id_t) + 8UL; + + struct vs_service_driver *vsdrv = + to_vs_service_driver(VS_STATE_SERVICE_PTR(_state)->dev.driver); + __maybe_unused struct vs_server_block *_server = + to_server_driver(vsdrv)->server; + + if (_opaque >= VSERVICE_BLOCK_IO_WRITE_MAX_PENDING) + return -EPROTO; + if (!VSERVICE_BASE_STATE_IS_RUNNING(_state->state.base)) + return -EPROTO; + if (!test_bit(_opaque, _state->state.io.write_bitmask)) + return -EPROTO; + + _mbuf = + vs_service_alloc_mbuf(VS_STATE_SERVICE_PTR(_state), _msg_size, + flags); + if (IS_ERR(_mbuf)) + return PTR_ERR(_mbuf); + if (!_mbuf) { + + WARN_ON_ONCE(1); + + return -ENOMEM; + } + + *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)) = + VSERVICE_BLOCK_IO_NACK_WRITE; + + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 0UL) = + _opaque; + *(vservice_block_block_io_error_t *) (VS_MBUF_DATA(_mbuf) + + sizeof(vs_message_id_t) + 4UL) = + err; + + { + int err = vs_service_send(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (err) { + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: Error %d sending message on transport.\n", + __func__, __LINE__, err); + + return err; + } + } + + __clear_bit(_opaque, _state->state.io.write_bitmask); + + return 0; +} + +EXPORT_SYMBOL(vs_server_block_io_send_nack_write); +static int +vs_server_block_io_handle_req_read(const struct vs_server_block *_server, + struct vs_server_block_state *_state, + struct vs_mbuf *_mbuf) +{ + const size_t _expected_size = sizeof(vs_message_id_t) + 24UL; + uint32_t _opaque; + uint64_t sector_index; + uint32_t num_sects; + bool nodelay; + bool flush; + + if (VS_MBUF_SIZE(_mbuf) < _expected_size) + return -EBADMSG; + + _opaque = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 0UL); + if (_state->state.base.statenum != VSERVICE_BASE_STATE_RUNNING) + return -EPROTO; + if (test_bit(_opaque, _state->state.io.read_bitmask)) + return -EPROTO; + __set_bit(_opaque, _state->state.io.read_bitmask); + _opaque = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 0UL); + sector_index = + *(uint64_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 4UL); + num_sects = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + + 12UL); + nodelay = + *(bool *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 16UL); + flush = + *(bool *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 20UL); + vs_service_free_mbuf(VS_STATE_SERVICE_PTR(_state), _mbuf); + if (_server->io.req_read) + return _server->io.req_read(_state, _opaque, sector_index, + num_sects, nodelay, flush); + else + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: No handler registered for _server->io.req_read, command will never be acknowledged\n", + __func__, __LINE__); + return 0; +} + +EXPORT_SYMBOL(vs_server_block_io_handle_req_read); +static int +vs_server_block_io_handle_req_write(const struct vs_server_block *_server, + struct vs_server_block_state *_state, + struct vs_mbuf *_mbuf) +{ + const size_t _max_size = + sizeof(vs_message_id_t) + _state->io.segment_size + 32UL; + uint32_t _opaque; + uint64_t sector_index; + uint32_t num_sects; + bool nodelay; + bool flush; + bool commit; + struct vs_pbuf data; + const size_t _min_size = _max_size - _state->io.segment_size; + size_t _exact_size; + + /* The first check is to ensure the message isn't complete garbage */ + if ((VS_MBUF_SIZE(_mbuf) > _max_size) + || (VS_MBUF_SIZE(_mbuf) < _min_size)) + return -EBADMSG; + _opaque = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 0UL); + if (_state->state.base.statenum != VSERVICE_BASE_STATE_RUNNING) + return -EPROTO; + if (test_bit(_opaque, _state->state.io.write_bitmask)) + return -EPROTO; + __set_bit(_opaque, _state->state.io.write_bitmask); + _opaque = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 0UL); + sector_index = + *(uint64_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 4UL); + num_sects = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + + 12UL); + nodelay = + *(bool *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 16UL); + flush = + *(bool *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 20UL); + commit = + *(bool *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + 24UL); + data.size = + *(uint32_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + + 28UL); + data.data = + (uintptr_t *) (VS_MBUF_DATA(_mbuf) + sizeof(vs_message_id_t) + + 28UL + sizeof(uint32_t)); + data.max_size = data.size; + + /* Now check the size received is the exact size expected */ + _exact_size = _max_size - (_state->io.segment_size - data.size); + if (VS_MBUF_SIZE(_mbuf) != _exact_size) + return -EBADMSG; + if (_server->io.req_write) + return _server->io.req_write(_state, _opaque, sector_index, + num_sects, nodelay, flush, commit, + data, _mbuf); + else + dev_warn(&_state->service->dev, + "[%s:%d] Protocol warning: No handler registered for _server->io.req_write, command will never be acknowledged\n", + __func__, __LINE__); + return 0; +} + +EXPORT_SYMBOL(vs_server_block_io_handle_req_write); +static int +block_handle_message(struct vs_service_device *service, struct vs_mbuf *_mbuf) +{ + vs_message_id_t message_id; + __maybe_unused struct vs_server_block_state *state = + dev_get_drvdata(&service->dev); + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + __maybe_unused struct vs_server_block *server = + to_server_driver(vsdrv)->server; + + int ret; + + /* Extract the message ID */ + if (VS_MBUF_SIZE(_mbuf) < sizeof(message_id)) { + dev_err(&state->service->dev, + "[%s:%d] Protocol error: Invalid message size %zd\n", + __func__, __LINE__, VS_MBUF_SIZE(_mbuf)); + + return -EBADMSG; + } + + message_id = *(vs_message_id_t *) (VS_MBUF_DATA(_mbuf)); + + switch (message_id) { + +/** interface base **/ +/* command in sync open */ + case VSERVICE_BLOCK_BASE_REQ_OPEN: + ret = vs_server_block_handle_req_open(server, state, _mbuf); + break; + +/* command in sync close */ + case VSERVICE_BLOCK_BASE_REQ_CLOSE: + ret = vs_server_block_handle_req_close(server, state, _mbuf); + break; + +/* command in sync reopen */ + case VSERVICE_BLOCK_BASE_REQ_REOPEN: + ret = vs_server_block_handle_req_reopen(server, state, _mbuf); + break; + +/** interface block_io **/ +/* command in parallel read */ + case VSERVICE_BLOCK_IO_REQ_READ: + ret = vs_server_block_io_handle_req_read(server, state, _mbuf); + break; + +/* command in parallel write */ + case VSERVICE_BLOCK_IO_REQ_WRITE: + ret = vs_server_block_io_handle_req_write(server, state, _mbuf); + break; + + default: + dev_err(&state->service->dev, + "[%s:%d] Protocol error: Unknown message type %d\n", + __func__, __LINE__, (int)message_id); + + ret = -EPROTO; + break; + } + + if (ret) { + dev_err(&state->service->dev, + "[%s:%d] Protocol error: Handler for message type %d returned %d\n", + __func__, __LINE__, (int)message_id, ret); + + } + + return ret; +} + +static void block_handle_notify(struct vs_service_device *service, + uint32_t notify_bits) +{ + __maybe_unused struct vs_server_block_state *state = + dev_get_drvdata(&service->dev); + struct vs_service_driver *vsdrv = + to_vs_service_driver(service->dev.driver); + __maybe_unused struct vs_server_block *server = + to_server_driver(vsdrv)->server; + + uint32_t bits = notify_bits; + int ret; + + while (bits) { + uint32_t not = __ffs(bits); + switch (not) { + + /** interface block_io **/ + + default: + dev_err(&state->service->dev, + "[%s:%d] Protocol error: Unknown notification %d\n", + __func__, __LINE__, (int)not); + + ret = -EPROTO; + break; + + } + bits &= ~(1 << not); + if (ret) { + dev_err(&state->service->dev, + "[%s:%d] Protocol error: Handler for notification %d returned %d\n", + __func__, __LINE__, (int)not, ret); + + } + } +} + +MODULE_DESCRIPTION("OKL4 Virtual Services blockServer Protocol Driver"); +MODULE_AUTHOR("Open Kernel Labs, Inc"); diff --git a/include/vservices/protocol/block/Kbuild b/include/vservices/protocol/block/Kbuild new file mode 100644 index 000000000000..ec3cbe813b00 --- /dev/null +++ b/include/vservices/protocol/block/Kbuild @@ -0,0 +1 @@ +header-y += types.h diff --git a/include/vservices/protocol/block/client.h b/include/vservices/protocol/block/client.h new file mode 100644 index 000000000000..4cd2847a74d5 --- /dev/null +++ b/include/vservices/protocol/block/client.h @@ -0,0 +1,175 @@ + +/* + * Copyright (c) 2012-2018 General Dynamics + * Copyright (c) 2014 Open Kernel Labs, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#if !defined(__VSERVICES_CLIENT_BLOCK__) +#define __VSERVICES_CLIENT_BLOCK__ + +struct vs_service_device; +struct vs_client_block_state; + +struct vs_client_block { + + /* + * If set to false then the receive message handlers are run from + * workqueue context and are allowed to sleep. If set to true the + * message handlers are run from tasklet context and may not sleep. + */ + bool rx_atomic; + + /* + * If this is set to true along with rx_atomic, the driver is allowed + * to send messages from softirq contexts other than the receive + * message handlers, after calling vs_service_state_lock_bh. Otherwise, + * messages may only be sent from the receive message handlers, or + * from task context after calling vs_service_state_lock. This must + * not be set to true if rx_atomic is set to false. + */ + bool tx_atomic; + /** session setup **/ + struct vs_client_block_state *(*alloc) (struct vs_service_device * + service); + void (*release) (struct vs_client_block_state * _state); + + struct vs_service_driver *driver; + +/** Opened, reopened and closed functions **/ + + void (*opened) (struct vs_client_block_state * _state); + + void (*reopened) (struct vs_client_block_state * _state); + + void (*closed) (struct vs_client_block_state * _state); + +/** Send/receive state callbacks **/ + int (*tx_ready) (struct vs_client_block_state * _state); + + struct { + int (*ack_read) (struct vs_client_block_state * _state, + void *_opaque, struct vs_pbuf data, + struct vs_mbuf * _mbuf); + int (*nack_read) (struct vs_client_block_state * _state, + void *_opaque, + vservice_block_block_io_error_t err); + + int (*ack_write) (struct vs_client_block_state * _state, + void *_opaque); + int (*nack_write) (struct vs_client_block_state * _state, + void *_opaque, + vservice_block_block_io_error_t err); + + } io; +}; + +struct vs_client_block_state { + vservice_block_state_t state; + bool readonly; + uint32_t sector_size; + uint32_t segment_size; + uint64_t device_sectors; + bool flushable; + bool committable; + struct { + uint32_t sector_size; + uint32_t segment_size; + } io; + struct vs_service_device *service; + bool released; +}; + +extern int vs_client_block_reopen(struct vs_client_block_state *_state); + +extern int vs_client_block_close(struct vs_client_block_state *_state); + + /** interface block_io **/ +/* command parallel read */ +extern int vs_client_block_io_getbufs_ack_read(struct vs_client_block_state + *_state, struct vs_pbuf *data, + struct vs_mbuf *_mbuf); +extern int vs_client_block_io_free_ack_read(struct vs_client_block_state + *_state, struct vs_pbuf *data, + struct vs_mbuf *_mbuf); +extern int vs_client_block_io_req_read(struct vs_client_block_state *_state, + void *_opaque, uint64_t sector_index, + uint32_t num_sects, bool nodelay, + bool flush, gfp_t flags); + + /* command parallel write */ +extern struct vs_mbuf *vs_client_block_io_alloc_req_write(struct + vs_client_block_state + *_state, + struct vs_pbuf *data, + gfp_t flags); +extern int vs_client_block_io_free_req_write(struct vs_client_block_state + *_state, struct vs_pbuf *data, + struct vs_mbuf *_mbuf); +extern int vs_client_block_io_req_write(struct vs_client_block_state *_state, + void *_opaque, uint64_t sector_index, + uint32_t num_sects, bool nodelay, + bool flush, bool commit, + struct vs_pbuf data, + struct vs_mbuf *_mbuf); + +/* Status APIs for async parallel commands */ +static inline bool vs_client_block_io_req_read_can_send(struct + vs_client_block_state + *_state) +{ + return !bitmap_full(_state->state.io.read_bitmask, + VSERVICE_BLOCK_IO_READ_MAX_PENDING); +} + +static inline bool vs_client_block_io_req_read_is_pending(struct + vs_client_block_state + *_state) +{ + return !bitmap_empty(_state->state.io.read_bitmask, + VSERVICE_BLOCK_IO_READ_MAX_PENDING); +} + +static inline bool vs_client_block_io_req_write_can_send(struct + vs_client_block_state + *_state) +{ + return !bitmap_full(_state->state.io.write_bitmask, + VSERVICE_BLOCK_IO_WRITE_MAX_PENDING); +} + +static inline bool vs_client_block_io_req_write_is_pending(struct + vs_client_block_state + *_state) +{ + return !bitmap_empty(_state->state.io.write_bitmask, + VSERVICE_BLOCK_IO_WRITE_MAX_PENDING); +} + +/** Module registration **/ + +struct module; + +extern int __vservice_block_client_register(struct vs_client_block *client, + const char *name, + struct module *owner); + +static inline int vservice_block_client_register(struct vs_client_block *client, + const char *name) +{ +#ifdef MODULE + extern struct module __this_module; + struct module *this_module = &__this_module; +#else + struct module *this_module = NULL; +#endif + + return __vservice_block_client_register(client, name, this_module); +} + +extern int vservice_block_client_unregister(struct vs_client_block *client); + +#endif /* ! __VSERVICES_CLIENT_BLOCK__ */ diff --git a/include/vservices/protocol/block/common.h b/include/vservices/protocol/block/common.h new file mode 100644 index 000000000000..2779b18783d0 --- /dev/null +++ b/include/vservices/protocol/block/common.h @@ -0,0 +1,42 @@ + +/* + * Copyright (c) 2012-2018 General Dynamics + * Copyright (c) 2014 Open Kernel Labs, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#if !defined(__VSERVICES_BLOCK_PROTOCOL_H__) +#define __VSERVICES_BLOCK_PROTOCOL_H__ + +#define VSERVICE_BLOCK_PROTOCOL_NAME "com.ok-labs.block" +typedef enum { + VSERVICE_BLOCK_BASE_REQ_OPEN, + VSERVICE_BLOCK_BASE_ACK_OPEN, + VSERVICE_BLOCK_BASE_NACK_OPEN, + VSERVICE_BLOCK_BASE_REQ_CLOSE, + VSERVICE_BLOCK_BASE_ACK_CLOSE, + VSERVICE_BLOCK_BASE_NACK_CLOSE, + VSERVICE_BLOCK_BASE_REQ_REOPEN, + VSERVICE_BLOCK_BASE_ACK_REOPEN, + VSERVICE_BLOCK_BASE_NACK_REOPEN, + VSERVICE_BLOCK_BASE_MSG_RESET, + VSERVICE_BLOCK_IO_REQ_READ, + VSERVICE_BLOCK_IO_ACK_READ, + VSERVICE_BLOCK_IO_NACK_READ, + VSERVICE_BLOCK_IO_REQ_WRITE, + VSERVICE_BLOCK_IO_ACK_WRITE, + VSERVICE_BLOCK_IO_NACK_WRITE, +} vservice_block_message_id_t; +typedef enum { + VSERVICE_BLOCK_NBIT_IN__COUNT +} vservice_block_nbit_in_t; + +typedef enum { + VSERVICE_BLOCK_NBIT_OUT__COUNT +} vservice_block_nbit_out_t; + +/* Notification mask macros */ +#endif /* ! __VSERVICES_BLOCK_PROTOCOL_H__ */ diff --git a/include/vservices/protocol/block/server.h b/include/vservices/protocol/block/server.h new file mode 100644 index 000000000000..65b0bfda162f --- /dev/null +++ b/include/vservices/protocol/block/server.h @@ -0,0 +1,177 @@ + +/* + * Copyright (c) 2012-2018 General Dynamics + * Copyright (c) 2014 Open Kernel Labs, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#if !defined(VSERVICES_SERVER_BLOCK) +#define VSERVICES_SERVER_BLOCK + +struct vs_service_device; +struct vs_server_block_state; + +struct vs_server_block { + + /* + * If set to false then the receive message handlers are run from + * workqueue context and are allowed to sleep. If set to true the + * message handlers are run from tasklet context and may not sleep. + */ + bool rx_atomic; + + /* + * If this is set to true along with rx_atomic, the driver is allowed + * to send messages from softirq contexts other than the receive + * message handlers, after calling vs_service_state_lock_bh. Otherwise, + * messages may only be sent from the receive message handlers, or + * from task context after calling vs_service_state_lock. This must + * not be set to true if rx_atomic is set to false. + */ + bool tx_atomic; + + /* + * These are the driver's recommended message quotas. They are used + * by the core service to select message quotas for services with no + * explicitly configured quotas. + */ + u32 in_quota_best; + u32 out_quota_best; + /** session setup **/ + struct vs_server_block_state *(*alloc) (struct vs_service_device * + service); + void (*release) (struct vs_server_block_state * _state); + + struct vs_service_driver *driver; + +/** Open, reopen, close and closed functions **/ + + vs_server_response_type_t(*open) (struct vs_server_block_state * + _state); + + vs_server_response_type_t(*reopen) (struct vs_server_block_state * + _state); + + vs_server_response_type_t(*close) (struct vs_server_block_state * + _state); + + void (*closed) (struct vs_server_block_state * _state); + +/** Send/receive state callbacks **/ + int (*tx_ready) (struct vs_server_block_state * _state); + + struct { + int (*req_read) (struct vs_server_block_state * _state, + uint32_t _opaque, uint64_t sector_index, + uint32_t num_sects, bool nodelay, bool flush); + + int (*req_write) (struct vs_server_block_state * _state, + uint32_t _opaque, uint64_t sector_index, + uint32_t num_sects, bool nodelay, bool flush, + bool commit, struct vs_pbuf data, + struct vs_mbuf * _mbuf); + + } io; +}; + +struct vs_server_block_state { + vservice_block_state_t state; + bool readonly; + uint32_t sector_size; + uint32_t segment_size; + uint64_t device_sectors; + bool flushable; + bool committable; + struct { + uint32_t sector_size; + uint32_t segment_size; + } io; + struct vs_service_device *service; + bool released; +}; + +/** Complete calls for server core functions **/ +extern int vs_server_block_open_complete(struct vs_server_block_state *_state, + vs_server_response_type_t resp); + +extern int vs_server_block_close_complete(struct vs_server_block_state *_state, + vs_server_response_type_t resp); + +extern int vs_server_block_reopen_complete(struct vs_server_block_state *_state, + vs_server_response_type_t resp); + + /** interface block_io **/ +/* command parallel read */ +extern struct vs_mbuf *vs_server_block_io_alloc_ack_read(struct + vs_server_block_state + *_state, + struct vs_pbuf *data, + gfp_t flags); +extern int vs_server_block_io_free_ack_read(struct vs_server_block_state + *_state, struct vs_pbuf *data, + struct vs_mbuf *_mbuf); +extern int vs_server_block_io_send_ack_read(struct vs_server_block_state + *_state, uint32_t _opaque, + struct vs_pbuf data, + struct vs_mbuf *_mbuf); +extern int vs_server_block_io_send_nack_read(struct vs_server_block_state + *_state, uint32_t _opaque, + vservice_block_block_io_error_t + err, gfp_t flags); + /* command parallel write */ +extern int vs_server_block_io_getbufs_req_write(struct vs_server_block_state + *_state, struct vs_pbuf *data, + struct vs_mbuf *_mbuf); +extern int vs_server_block_io_free_req_write(struct vs_server_block_state + *_state, struct vs_pbuf *data, + struct vs_mbuf *_mbuf); +extern int vs_server_block_io_send_ack_write(struct vs_server_block_state + *_state, uint32_t _opaque, + gfp_t flags); +extern int vs_server_block_io_send_nack_write(struct vs_server_block_state + *_state, uint32_t _opaque, + vservice_block_block_io_error_t + err, gfp_t flags); + +static inline bool vs_server_block_io_send_ack_read_is_pending(struct + vs_server_block_state + *_state) +{ + return !bitmap_empty(_state->state.io.read_bitmask, + VSERVICE_BLOCK_IO_READ_MAX_PENDING); +} + +static inline bool vs_server_block_io_send_ack_write_is_pending(struct + vs_server_block_state + *_state) +{ + return !bitmap_empty(_state->state.io.write_bitmask, + VSERVICE_BLOCK_IO_WRITE_MAX_PENDING); +} + +/** Module registration **/ + +struct module; + +extern int __vservice_block_server_register(struct vs_server_block *server, + const char *name, + struct module *owner); + +static inline int vservice_block_server_register(struct vs_server_block *server, + const char *name) +{ +#ifdef MODULE + extern struct module __this_module; + struct module *this_module = &__this_module; +#else + struct module *this_module = NULL; +#endif + + return __vservice_block_server_register(server, name, this_module); +} + +extern int vservice_block_server_unregister(struct vs_server_block *server); +#endif /* ! VSERVICES_SERVER_BLOCK */ diff --git a/include/vservices/protocol/block/types.h b/include/vservices/protocol/block/types.h new file mode 100644 index 000000000000..52845a3564ef --- /dev/null +++ b/include/vservices/protocol/block/types.h @@ -0,0 +1,106 @@ + +/* + * Copyright (c) 2012-2018 General Dynamics + * Copyright (c) 2014 Open Kernel Labs, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#if !defined(VSERVICES_BLOCK_TYPES_H) +#define VSERVICES_BLOCK_TYPES_H + +#define VSERVICE_BLOCK_IO_READ_MAX_PENDING 1024 +#define VSERVICE_BLOCK_IO_WRITE_MAX_PENDING 1024 + +typedef enum vservice_block_block_io_error { + VSERVICE_BLOCK_INVALID_INDEX, + VSERVICE_BLOCK_MEDIA_FAILURE, + VSERVICE_BLOCK_MEDIA_TIMEOUT, + VSERVICE_BLOCK_UNSUPPORTED_COMMAND, + VSERVICE_BLOCK_SERVICE_RESET +} vservice_block_block_io_error_t; + +typedef enum { +/* state closed */ + VSERVICE_BASE_STATE_CLOSED = 0, + VSERVICE_BASE_STATE_CLOSED__OPEN, + VSERVICE_BASE_STATE_CLOSED__CLOSE, + VSERVICE_BASE_STATE_CLOSED__REOPEN, + +/* state running */ + VSERVICE_BASE_STATE_RUNNING, + VSERVICE_BASE_STATE_RUNNING__OPEN, + VSERVICE_BASE_STATE_RUNNING__CLOSE, + VSERVICE_BASE_STATE_RUNNING__REOPEN, + + VSERVICE_BASE__RESET = VSERVICE_BASE_STATE_CLOSED +} vservice_base_statenum_t; + +typedef struct { + vservice_base_statenum_t statenum; +} vservice_base_state_t; + +#define VSERVICE_BASE_RESET_STATE (vservice_base_state_t) { \ +.statenum = VSERVICE_BASE__RESET} + +#define VSERVICE_BASE_STATE_IS_CLOSED(state) (\ +((state).statenum == VSERVICE_BASE_STATE_CLOSED) || \ +((state).statenum == VSERVICE_BASE_STATE_CLOSED__OPEN) || \ +((state).statenum == VSERVICE_BASE_STATE_CLOSED__CLOSE) || \ +((state).statenum == VSERVICE_BASE_STATE_CLOSED__REOPEN)) + +#define VSERVICE_BASE_STATE_IS_RUNNING(state) (\ +((state).statenum == VSERVICE_BASE_STATE_RUNNING) || \ +((state).statenum == VSERVICE_BASE_STATE_RUNNING__OPEN) || \ +((state).statenum == VSERVICE_BASE_STATE_RUNNING__CLOSE) || \ +((state).statenum == VSERVICE_BASE_STATE_RUNNING__REOPEN)) + +#define VSERVICE_BASE_STATE_VALID(state) ( \ +VSERVICE_BASE_STATE_IS_CLOSED(state) ? true : \ +VSERVICE_BASE_STATE_IS_RUNNING(state) ? true : \ +false) + +static inline const char *vservice_base_get_state_string(vservice_base_state_t + state) +{ + static const char *names[] = + { "closed", "closed__open", "closed__close", "closed__reopen", + "running", "running__open", "running__close", "running__reopen" + }; + if (!VSERVICE_BASE_STATE_VALID(state)) { + return "INVALID"; + } + return names[state.statenum]; +} + +typedef struct { + DECLARE_BITMAP(read_bitmask, VSERVICE_BLOCK_IO_READ_MAX_PENDING); + void *read_tags[VSERVICE_BLOCK_IO_READ_MAX_PENDING]; + DECLARE_BITMAP(write_bitmask, VSERVICE_BLOCK_IO_WRITE_MAX_PENDING); + void *write_tags[VSERVICE_BLOCK_IO_WRITE_MAX_PENDING]; +} vservice_block_io_state_t; + +#define VSERVICE_BLOCK_IO_RESET_STATE (vservice_block_io_state_t) { \ +.read_bitmask = {0}, \ +.read_tags = {NULL}, \ +.write_bitmask = {0}, \ +.write_tags = {NULL}} + +#define VSERVICE_BLOCK_IO_STATE_VALID(state) true + +typedef struct { + + vservice_base_state_t base; + + vservice_block_io_state_t io; +} vservice_block_state_t; + +#define VSERVICE_BLOCK_RESET_STATE (vservice_block_state_t) {\ +.base = VSERVICE_BASE_RESET_STATE,\ +.io = VSERVICE_BLOCK_IO_RESET_STATE } + +#define VSERVICE_BLOCK_IS_STATE_RESET(state) \ + ((state).base.statenum == VSERVICE_BASE__RESET) +#endif /* ! VSERVICES_BLOCK_TYPES_H */