From: Milan Broz <mbroz@redhat.com> Subject: [RHEL 5.1 PATCH] dm: failures when creating many snapshots (new dm-io interface) Date: Tue, 24 Apr 2007 15:57:14 +0200 Bugzilla: 211516 211525 Message-Id: <462E0CBA.8080602@redhat.com> Changelog: [dm] failures when creating many snapshots RHEL5.1 device mapper: failures when creating many snapshots (new dm-io interface) Resolves: rhbz#211516 - memory issues cause "failures" when creating many snapshots rhbz#211525 - mempool_resize BUG() during multiple snapshot removals Current implementation of dm-io uses one common mempool for all clients. This causes many problems when kernel cannot alocate or resize such big amount of memory. These patches backport a new API to dm-io that uses a private mempool and bio_set for each client. All dm internal modules are patched to use new dm-io interface (kcopyd, exception store, raid1). New interface is also required for dm-raid45 target. Patch retaining full compatibility with old interface, no changes are needed for external modules. kABI impact: these exported functions are added (new dm-io interface): dm_io_client_create dm_io_client_resize dm_io_client_destroy dm_io No changes to existing kABI, just these existing functions use pointer to changed internal struct (hidden by ifdef magic) kcopyd_client_create kcopyd_client_destroy Patches are upstream in -mm tree (planned for 2.6.22). Testing: - All targets using dm-io tested with new patch including pvmove/mirror operation + integrity test after move, snapshot creation an removing. - There are test cases on bugzillas to reproduce problems with multiple snapshots. --- drivers/md/dm-exception-store.c | 49 +++++---- drivers/md/dm-io.c | 204 +++++++++++++++++++++++++++++++++++----- drivers/md/dm-io.h | 53 ++++++++++ drivers/md/dm-log.c | 31 ++++-- drivers/md/dm-raid1.c | 43 ++++++-- drivers/md/kcopyd.c | 28 +++-- 6 files changed, 334 insertions(+), 74 deletions(-) Index: linux-2.6.18/drivers/md/dm-io.c =================================================================== --- linux-2.6.18.orig/drivers/md/dm-io.c 2007-04-23 14:46:44.000000000 +0200 +++ linux-2.6.18/drivers/md/dm-io.c 2007-04-23 14:49:27.000000000 +0200 @@ -14,11 +14,17 @@ static struct bio_set *_bios; +struct dm_io_client { + mempool_t *pool; + struct bio_set *bios; +}; + /* FIXME: can we shrink this ? */ struct io { unsigned long error; atomic_t count; struct task_struct *sleeper; + struct dm_io_client *client; io_notify_fn callback; void *context; }; @@ -32,6 +38,19 @@ struct io { static unsigned _num_ios; static mempool_t *_io_pool; +/* + * Temporary functions to allow old and new interfaces to co-exist. + */ +static struct bio_set *bios(struct dm_io_client *client) +{ + return client ? client->bios : _bios; +} + +static mempool_t *io_pool(struct dm_io_client *client) +{ + return client ? client->pool : _io_pool; +} + static unsigned int pages_to_ios(unsigned int pages) { return 4 * pages; /* too many ? */ @@ -84,6 +103,51 @@ void dm_io_put(unsigned int num_pages) resize_pool(_num_ios - pages_to_ios(num_pages)); } +/* + * Create a client with mempool and bioset. + */ +struct dm_io_client *dm_io_client_create(unsigned num_pages) +{ + unsigned ios = pages_to_ios(num_pages); + struct dm_io_client *client; + + client = kmalloc(sizeof(*client), GFP_KERNEL); + if (!client) + return ERR_PTR(-ENOMEM); + + client->pool = mempool_create_kmalloc_pool(ios, sizeof(struct io)); + if (!client->pool) + goto bad; + + client->bios = bioset_create(16, 16, 4); + if (!client->bios) + goto bad; + + return client; + + bad: + if (client->pool) + mempool_destroy(client->pool); + kfree(client); + return ERR_PTR(-ENOMEM); +} +EXPORT_SYMBOL(dm_io_client_create); + +int dm_io_client_resize(unsigned num_pages, struct dm_io_client *client) +{ + return mempool_resize(client->pool, pages_to_ios(num_pages), + GFP_KERNEL); +} +EXPORT_SYMBOL(dm_io_client_resize); + +void dm_io_client_destroy(struct dm_io_client *client) +{ + mempool_destroy(client->pool); + bioset_free(client->bios); + kfree(client); +} +EXPORT_SYMBOL(dm_io_client_destroy); + /*----------------------------------------------------------------- * We need to keep track of which region a bio is doing io for. * In order to save a memory allocation we store this the last @@ -92,12 +156,12 @@ void dm_io_put(unsigned int num_pages) *---------------------------------------------------------------*/ static inline void bio_set_region(struct bio *bio, unsigned region) { - bio->bi_io_vec[bio->bi_max_vecs - 1].bv_len = region; + bio->bi_io_vec[bio->bi_max_vecs].bv_len = region; } static inline unsigned bio_get_region(struct bio *bio) { - return bio->bi_io_vec[bio->bi_max_vecs - 1].bv_len; + return bio->bi_io_vec[bio->bi_max_vecs].bv_len; } /*----------------------------------------------------------------- @@ -118,7 +182,7 @@ static void dec_count(struct io *io, uns io_notify_fn fn = io->callback; void *context = io->context; - mempool_free(io, _io_pool); + mempool_free(io, io_pool(io->client)); fn(r, context); } } @@ -126,7 +190,8 @@ static void dec_count(struct io *io, uns static int endio(struct bio *bio, unsigned int done, int error) { - struct io *io = (struct io *) bio->bi_private; + struct io *io; + unsigned region; /* keep going until we've finished */ if (bio->bi_size) @@ -135,9 +200,17 @@ static int endio(struct bio *bio, unsign if (error && bio_data_dir(bio) == READ) zero_fill_bio(bio); - dec_count(io, bio_get_region(bio), error); + /* + * The bio destructor in bio_put() may use the io object. + */ + io = bio->bi_private; + region = bio_get_region(bio); + + bio->bi_max_vecs++; bio_put(bio); + dec_count(io, region, error); + return 0; } @@ -208,6 +281,9 @@ static void bvec_dp_init(struct dpages * dp->context_ptr = bvec; } +/* + * Functions for getting the pages from a VMA. + */ static void vm_get_page(struct dpages *dp, struct page **p, unsigned long *len, unsigned *offset) { @@ -232,7 +308,34 @@ static void vm_dp_init(struct dpages *dp static void dm_bio_destructor(struct bio *bio) { - bio_free(bio, _bios); + struct io *io = bio->bi_private; + + bio_free(bio, bios(io->client)); +} + +/* + * Functions for getting the pages from kernel memory. + */ +static void km_get_page(struct dpages *dp, struct page **p, unsigned long *len, + unsigned *offset) +{ + *p = virt_to_page(dp->context_ptr); + *offset = dp->context_u; + *len = PAGE_SIZE - dp->context_u; +} + +static void km_next_page(struct dpages *dp) +{ + dp->context_ptr += PAGE_SIZE - dp->context_u; + dp->context_u = 0; +} + +static void km_dp_init(struct dpages *dp, void *data) +{ + dp->get_page = km_get_page; + dp->next_page = km_next_page; + dp->context_u = ((unsigned long) data) & (PAGE_SIZE - 1); + dp->context_ptr = data; } /*----------------------------------------------------------------- @@ -250,16 +353,18 @@ static void do_region(int rw, unsigned i while (remaining) { /* - * Allocate a suitably sized bio, we add an extra - * bvec for bio_get/set_region(). + * Allocate a suitably sized-bio: we add an extra + * bvec for bio_get/set_region() and decrement bi_max_vecs + * to hide it from bio_add_page(). */ - num_bvecs = (remaining / (PAGE_SIZE >> 9)) + 2; - bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, _bios); + num_bvecs = (remaining / (PAGE_SIZE >> SECTOR_SHIFT)) + 2; + bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, bios(io->client)); bio->bi_sector = where->sector + (where->count - remaining); bio->bi_bdev = where->bdev; bio->bi_end_io = endio; bio->bi_private = io; bio->bi_destructor = dm_bio_destructor; + bio->bi_max_vecs--; bio_set_region(bio, region); /* @@ -308,8 +413,9 @@ static void dispatch_io(int rw, unsigned dec_count(io, 0, 0); } -static int sync_io(unsigned int num_regions, struct io_region *where, - int rw, struct dpages *dp, unsigned long *error_bits) +static int sync_io(struct dm_io_client *client, unsigned int num_regions, + struct io_region *where, int rw, struct dpages *dp, + unsigned long *error_bits) { struct io io; @@ -321,6 +427,7 @@ static int sync_io(unsigned int num_regi io.error = 0; atomic_set(&io.count, 1); /* see dispatch_io() */ io.sleeper = current; + io.client = client; dispatch_io(rw, num_regions, where, dp, &io, 1); @@ -337,12 +444,15 @@ static int sync_io(unsigned int num_regi if (atomic_read(&io.count)) return -EINTR; - *error_bits = io.error; + if (error_bits) + *error_bits = io.error; + return io.error ? -EIO : 0; } -static int async_io(unsigned int num_regions, struct io_region *where, int rw, - struct dpages *dp, io_notify_fn fn, void *context) +static int async_io(struct dm_io_client *client, unsigned int num_regions, + struct io_region *where, int rw, struct dpages *dp, + io_notify_fn fn, void *context) { struct io *io; @@ -352,10 +462,11 @@ static int async_io(unsigned int num_reg return -EIO; } - io = mempool_alloc(_io_pool, GFP_NOIO); + io = mempool_alloc(io_pool(client), GFP_NOIO); io->error = 0; atomic_set(&io->count, 1); /* see dispatch_io() */ io->sleeper = NULL; + io->client = client; io->callback = fn; io->context = context; @@ -369,7 +480,7 @@ int dm_io_sync(unsigned int num_regions, { struct dpages dp; list_dp_init(&dp, pl, offset); - return sync_io(num_regions, where, rw, &dp, error_bits); + return sync_io(NULL, num_regions, where, rw, &dp, error_bits); } int dm_io_sync_bvec(unsigned int num_regions, struct io_region *where, int rw, @@ -377,7 +488,7 @@ int dm_io_sync_bvec(unsigned int num_reg { struct dpages dp; bvec_dp_init(&dp, bvec); - return sync_io(num_regions, where, rw, &dp, error_bits); + return sync_io(NULL, num_regions, where, rw, &dp, error_bits); } int dm_io_sync_vm(unsigned int num_regions, struct io_region *where, int rw, @@ -385,7 +496,7 @@ int dm_io_sync_vm(unsigned int num_regio { struct dpages dp; vm_dp_init(&dp, data); - return sync_io(num_regions, where, rw, &dp, error_bits); + return sync_io(NULL, num_regions, where, rw, &dp, error_bits); } int dm_io_async(unsigned int num_regions, struct io_region *where, int rw, @@ -394,7 +505,7 @@ int dm_io_async(unsigned int num_regions { struct dpages dp; list_dp_init(&dp, pl, offset); - return async_io(num_regions, where, rw, &dp, fn, context); + return async_io(NULL, num_regions, where, rw, &dp, fn, context); } int dm_io_async_bvec(unsigned int num_regions, struct io_region *where, int rw, @@ -402,7 +513,7 @@ int dm_io_async_bvec(unsigned int num_re { struct dpages dp; bvec_dp_init(&dp, bvec); - return async_io(num_regions, where, rw, &dp, fn, context); + return async_io(NULL, num_regions, where, rw, &dp, fn, context); } int dm_io_async_vm(unsigned int num_regions, struct io_region *where, int rw, @@ -410,8 +521,57 @@ int dm_io_async_vm(unsigned int num_regi { struct dpages dp; vm_dp_init(&dp, data); - return async_io(num_regions, where, rw, &dp, fn, context); + return async_io(NULL, num_regions, where, rw, &dp, fn, context); +} + +static int dp_init(struct dm_io_request *io_req, struct dpages *dp) +{ + /* Set up dpages based on memory type */ + switch (io_req->mem.type) { + case DM_IO_PAGE_LIST: + list_dp_init(dp, io_req->mem.ptr.pl, io_req->mem.offset); + break; + + case DM_IO_BVEC: + bvec_dp_init(dp, io_req->mem.ptr.bvec); + break; + + case DM_IO_VMA: + vm_dp_init(dp, io_req->mem.ptr.vma); + break; + + case DM_IO_KMEM: + km_dp_init(dp, io_req->mem.ptr.addr); + break; + + default: + return -EINVAL; + } + + return 0; +} + +/* + * New collapsed (a)synchronous interface + */ +int dm_io(struct dm_io_request *io_req, unsigned num_regions, + struct io_region *where, unsigned long *sync_error_bits) +{ + int r; + struct dpages dp; + + r = dp_init(io_req, &dp); + if (r) + return r; + + if (!io_req->notify.fn) + return sync_io(io_req->client, num_regions, where, + io_req->bi_rw, &dp, sync_error_bits); + + return async_io(io_req->client, num_regions, where, io_req->bi_rw, + &dp, io_req->notify.fn, io_req->notify.context); } +EXPORT_SYMBOL(dm_io); EXPORT_SYMBOL(dm_io_get); EXPORT_SYMBOL(dm_io_put); Index: linux-2.6.18/drivers/md/dm-io.h =================================================================== --- linux-2.6.18.orig/drivers/md/dm-io.h 2007-04-23 14:46:44.000000000 +0200 +++ linux-2.6.18/drivers/md/dm-io.h 2007-04-23 15:17:20.000000000 +0200 @@ -27,6 +27,41 @@ struct page_list { */ typedef void (*io_notify_fn)(unsigned long error, void *context); +enum dm_io_mem_type { + DM_IO_PAGE_LIST,/* Page list */ + DM_IO_BVEC, /* Bio vector */ + DM_IO_VMA, /* Virtual memory area */ + DM_IO_KMEM, /* Kernel memory */ +}; + +struct dm_io_memory { + enum dm_io_mem_type type; + + union { + struct page_list *pl; + struct bio_vec *bvec; + void *vma; + void *addr; + } ptr; + + unsigned offset; +}; + +struct dm_io_notify { + io_notify_fn fn; /* Callback for asynchronous requests */ + void *context; /* Passed to callback */ +}; + +/* + * IO request structure + */ +struct dm_io_client; +struct dm_io_request { + int bi_rw; /* READ|WRITE - not READA */ + struct dm_io_memory mem; /* Memory to use for io */ + struct dm_io_notify notify; /* Synchronous if notify.fn is NULL */ + struct dm_io_client *client; /* Client memory handler */ +}; /* * Before anyone uses the IO interface they should call @@ -39,6 +74,16 @@ int dm_io_get(unsigned int num_pages); void dm_io_put(unsigned int num_pages); /* + * For async io calls, users can alternatively use the dm_io() function below + * and dm_io_client_create() to create private mempools for the client. + * + * Create/destroy may block. + */ +struct dm_io_client *dm_io_client_create(unsigned num_pages); +int dm_io_client_resize(unsigned num_pages, struct dm_io_client *client); +void dm_io_client_destroy(struct dm_io_client *client); + +/* * Synchronous IO. * * Please ensure that the rw flag in the next two functions is @@ -71,4 +116,12 @@ int dm_io_async_bvec(unsigned int num_re int dm_io_async_vm(unsigned int num_regions, struct io_region *where, int rw, void *data, io_notify_fn fn, void *context); +/* + * IO interface using private per-client pools. + * Each bit in the optional 'sync_error_bits' bitset indicates whether an + * error occurred doing io to the corresponding region. + */ +int dm_io(struct dm_io_request *io_req, unsigned num_regions, + struct io_region *region, unsigned long *sync_error_bits); + #endif Index: linux-2.6.18/drivers/md/kcopyd.c =================================================================== --- linux-2.6.18.orig/drivers/md/kcopyd.c 2007-04-23 14:46:44.000000000 +0200 +++ linux-2.6.18/drivers/md/kcopyd.c 2007-04-23 15:09:03.000000000 +0200 @@ -47,6 +47,9 @@ struct kcopyd_client { wait_queue_head_t destroyq; atomic_t nr_jobs; +#ifndef __GENKSYMS__ + struct dm_io_client *io_client; +#endif }; static struct page_list *alloc_pl(void) @@ -342,16 +345,20 @@ static void complete_io(unsigned long er static int run_io_job(struct kcopyd_job *job) { int r; + struct dm_io_request io_req = { + .bi_rw = job->rw, + .mem.type = DM_IO_PAGE_LIST, + .mem.ptr.pl = job->pages, + .mem.offset = job->offset, + .notify.fn = complete_io, + .notify.context = job, + .client = job->kc->io_client, + }; if (job->rw == READ) - r = dm_io_async(1, &job->source, job->rw, - job->pages, - job->offset, complete_io, job); - + r = dm_io(&io_req, 1, &job->source, NULL); else - r = dm_io_async(job->num_dests, job->dests, job->rw, - job->pages, - job->offset, complete_io, job); + r = dm_io(&io_req, job->num_dests, job->dests, NULL); return r; } @@ -670,8 +677,9 @@ int kcopyd_client_create(unsigned int nr return r; } - r = dm_io_get(nr_pages); - if (r) { + kc->io_client = dm_io_client_create(nr_pages); + if (IS_ERR(kc->io_client)) { + r = PTR_ERR(kc->io_client); client_free_pages(kc); kfree(kc); kcopyd_exit(); @@ -691,7 +699,7 @@ void kcopyd_client_destroy(struct kcopyd /* Wait for completion of all jobs submitted by this client. */ wait_event(kc->destroyq, !atomic_read(&kc->nr_jobs)); - dm_io_put(kc->nr_pages); + dm_io_client_destroy(kc->io_client); client_free_pages(kc); client_del(kc); kfree(kc); Index: linux-2.6.18/drivers/md/dm-exception-store.c =================================================================== --- linux-2.6.18.orig/drivers/md/dm-exception-store.c 2007-04-23 14:46:44.000000000 +0200 +++ linux-2.6.18/drivers/md/dm-exception-store.c 2007-04-23 14:49:27.000000000 +0200 @@ -123,6 +123,7 @@ struct pstore { atomic_t pending_count; uint32_t callback_count; struct commit_callback *callbacks; + struct dm_io_client *io_client; }; static inline unsigned int sectors_to_pages(unsigned int sectors) @@ -159,14 +160,20 @@ static void free_area(struct pstore *ps) */ static int chunk_io(struct pstore *ps, uint32_t chunk, int rw) { - struct io_region where; - unsigned long bits; + struct io_region where = { + .bdev = ps->snap->cow->bdev, + .sector = ps->snap->chunk_size * chunk, + .count = ps->snap->chunk_size, + }; + struct dm_io_request io_req = { + .bi_rw = rw, + .mem.type = DM_IO_VMA, + .mem.ptr.vma = ps->area, + .client = ps->io_client, + .notify.fn = NULL, + }; - where.bdev = ps->snap->cow->bdev; - where.sector = ps->snap->chunk_size * chunk; - where.count = ps->snap->chunk_size; - - return dm_io_sync_vm(1, &where, rw, ps->area, &bits); + return dm_io(&io_req, 1, &where, NULL); } /* @@ -213,17 +220,18 @@ static int read_header(struct pstore *ps chunk_size_supplied = 0; } - r = dm_io_get(sectors_to_pages(ps->snap->chunk_size)); - if (r) - return r; + ps->io_client = dm_io_client_create(sectors_to_pages(ps->snap-> + chunk_size)); + if (IS_ERR(ps->io_client)) + return PTR_ERR(ps->io_client); r = alloc_area(ps); if (r) - goto bad1; + return r; r = chunk_io(ps, 0, READ); if (r) - goto bad2; + goto bad; dh = (struct disk_header *) ps->area; @@ -235,7 +243,7 @@ static int read_header(struct pstore *ps if (le32_to_cpu(dh->magic) != SNAP_MAGIC) { DMWARN("Invalid or corrupt snapshot"); r = -ENXIO; - goto bad2; + goto bad; } *new_snapshot = 0; @@ -252,27 +260,22 @@ static int read_header(struct pstore *ps (unsigned long long)ps->snap->chunk_size); /* We had a bogus chunk_size. Fix stuff up. */ - dm_io_put(sectors_to_pages(ps->snap->chunk_size)); free_area(ps); ps->snap->chunk_size = chunk_size; ps->snap->chunk_mask = chunk_size - 1; ps->snap->chunk_shift = ffs(chunk_size) - 1; - r = dm_io_get(sectors_to_pages(chunk_size)); + r = dm_io_client_resize(sectors_to_pages(ps->snap->chunk_size), + ps->io_client); if (r) return r; r = alloc_area(ps); - if (r) - goto bad1; - - return 0; + return r; -bad2: +bad: free_area(ps); -bad1: - dm_io_put(sectors_to_pages(ps->snap->chunk_size)); return r; } @@ -405,7 +408,7 @@ static void persistent_destroy(struct ex { struct pstore *ps = get_info(store); - dm_io_put(sectors_to_pages(ps->snap->chunk_size)); + dm_io_client_destroy(ps->io_client); vfree(ps->callbacks); free_area(ps); kfree(ps); Index: linux-2.6.18/drivers/md/dm-log.c =================================================================== --- linux-2.6.18.orig/drivers/md/dm-log.c 2007-04-23 14:46:44.000000000 +0200 +++ linux-2.6.18/drivers/md/dm-log.c 2007-04-23 14:49:27.000000000 +0200 @@ -152,6 +152,8 @@ struct log_c { int failure_response; + struct dm_io_request io_req; + /* * Disk log fields */ @@ -203,13 +205,20 @@ static void header_from_disk(struct log_ core->nr_regions = le64_to_cpu(disk->nr_regions); } +static int rw_header(struct log_c *lc, int rw) +{ + lc->io_req.bi_rw = rw; + lc->io_req.mem.ptr.vma = lc->disk_header; + lc->io_req.notify.fn = NULL; + + return dm_io(&lc->io_req, 1, &lc->header_location, NULL); +} + static int read_header(struct log_c *log) { int r; - unsigned long ebits; - r = dm_io_sync_vm(1, &log->header_location, READ, - log->disk_header, &ebits); + r = rw_header(log, READ); if (r) return r; @@ -237,11 +246,8 @@ static int read_header(struct log_c *log static inline int write_header(struct log_c *log) { - unsigned long ebits; - header_to_disk(&log->header, log->disk_header); - return dm_io_sync_vm(1, &log->header_location, WRITE, - log->disk_header, &ebits); + return rw_header(log, WRITE); } /*---------------------------------------------------------------- @@ -262,6 +268,7 @@ static int create_log_context(struct dir uint32_t region_size; unsigned int region_count; size_t bitset_size, buf_size; + int r; if (argc < 1 || argc > 3) { DMWARN("wrong number of arguments to mirror log"); @@ -334,6 +341,15 @@ static int create_log_context(struct dir buf_size = dm_round_up((LOG_OFFSET << SECTOR_SHIFT) + bitset_size, ti->limits.hardsect_size); lc->header_location.count = buf_size >> SECTOR_SHIFT; + lc->io_req.mem.type = DM_IO_VMA; + lc->io_req.client = dm_io_client_create(dm_div_up(buf_size, + PAGE_SIZE)); + if (IS_ERR(lc->io_req.client)) { + r = PTR_ERR(lc->io_req.client); + DMWARN("couldn't allocate disk io client"); + kfree(lc); + return -ENOMEM; + } lc->disk_header = vmalloc(buf_size); if (!lc->disk_header) { @@ -435,6 +451,7 @@ static void disk_dtr(struct dirty_log *l dm_put_device(lc->ti, lc->log_dev); vfree(lc->disk_header); + dm_io_client_destroy(lc->io_req.client); destroy_log_context(lc); } Index: linux-2.6.18/drivers/md/dm-raid1.c =================================================================== --- linux-2.6.18.orig/drivers/md/dm-raid1.c 2007-04-23 14:46:44.000000000 +0200 +++ linux-2.6.18/drivers/md/dm-raid1.c 2007-04-23 14:49:27.000000000 +0200 @@ -22,6 +22,7 @@ #include <linux/workqueue.h> #define DM_MSG_PREFIX "raid1" +#define DM_IO_PAGES 64 DECLARE_WAIT_QUEUE_HEAD(recovery_stopped_event); @@ -129,6 +130,8 @@ struct mirror_set { struct bio_list writes; struct bio_list failures; + struct dm_io_client *io_client; + /* recovery */ region_t nr_regions; int in_sync; @@ -937,12 +940,18 @@ static void read_callback(unsigned long static void read_async_bio(struct mirror *m, struct bio *bio) { struct io_region io; + struct dm_io_request io_req = { + .bi_rw = READ, + .mem.type = DM_IO_BVEC, + .mem.ptr.bvec = bio->bi_io_vec + bio->bi_idx, + .notify.fn = read_callback, + .notify.context = bio, + .client = m->ms->io_client, + }; map_region(&io, m, bio); bio_set_m(bio, m); - dm_io_async_bvec(1, &io, READ, - bio->bi_io_vec + bio->bi_idx, - read_callback, bio); + (void) dm_io(&io_req, m->ms->nr_mirrors, &io, NULL); } static void do_reads(struct mirror_set *ms, struct bio_list *reads) @@ -1108,6 +1117,14 @@ static void do_write(struct mirror_set * unsigned int i; struct io_region io[ms->nr_mirrors], *dest = io; struct mirror *m; + struct dm_io_request io_req = { + .bi_rw = WRITE, + .mem.type = DM_IO_BVEC, + .mem.ptr.bvec = bio->bi_io_vec + bio->bi_idx, + .notify.fn = NULL, + .notify.context = bio, + .client = ms->io_client, + }; if (log_failure && dm_mirror_error_on_log_failure) { bio_endio(bio, bio->bi_size, -EIO); @@ -1123,14 +1140,9 @@ static void do_write(struct mirror_set * * to the mirror set in write_callback(). */ bio_set_m(bio, ms->default_mirror); - if (log_failure) - dm_io_async_bvec(ms->nr_mirrors, io, WRITE, - bio->bi_io_vec + bio->bi_idx, - write_callback_bad_log, bio); - else - dm_io_async_bvec(ms->nr_mirrors, io, WRITE, - bio->bi_io_vec + bio->bi_idx, - write_callback_good_log, bio); + io_req.notify.fn = log_failure ? write_callback_bad_log : + write_callback_good_log; + (void) dm_io(&io_req, ms->nr_mirrors, io, NULL); } static void do_writes(struct mirror_set *ms, struct bio_list *writes) @@ -1294,6 +1306,13 @@ static struct mirror_set *alloc_context( ms->read_mirror = &ms->mirror[DEFAULT_MIRROR]; ms->default_mirror = &ms->mirror[DEFAULT_MIRROR]; + ms->io_client = dm_io_client_create(DM_IO_PAGES); + if (IS_ERR(ms->io_client)) { + ti->error = "Error creating dm_io client"; + kfree(ms); + return NULL; + } + if (rh_init(&ms->rh, ms, dl, region_size, ms->nr_regions)) { ti->error = "Error creating dirty region hash"; kfree(ms); @@ -1313,6 +1332,7 @@ static void free_context(struct mirror_s while (m--) dm_put_device(ti, ms->mirror[m].dev); + dm_io_client_destroy(ms->io_client); rh_exit(&ms->rh); kfree(ms); } @@ -1398,7 +1418,6 @@ static struct dirty_log *create_dirty_lo * log_type is "core" or "disk" * #log_params is between 1 and 3 */ -#define DM_IO_PAGES 64 static int mirror_ctr(struct dm_target *ti, unsigned int argc, char **argv) { int r;