Update bcachefs sources to e9e735c1cda1 bcachefs: kill off bvec.bv_page uses

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

.bcachefs_revision

@@ -1 +1 @@
-47854fc052affa886552a188272cff4e08a7582f
+e9e735c1cda1484f80cda34669849f6a02053cb6

include/linux/bio.h

@@ -34,15 +34,9 @@
 #define bio_iter_iovec(bio, iter)				\
 	bvec_iter_bvec((bio)->bi_io_vec, (iter))
 
-#define bio_iter_page(bio, iter)				\
-	bvec_iter_page((bio)->bi_io_vec, (iter))
 #define bio_iter_len(bio, iter)					\
 	bvec_iter_len((bio)->bi_io_vec, (iter))
-#define bio_iter_offset(bio, iter)				\
-	bvec_iter_offset((bio)->bi_io_vec, (iter))
 
-#define bio_page(bio)		bio_iter_page((bio), (bio)->bi_iter)
-#define bio_offset(bio)		bio_iter_offset((bio), (bio)->bi_iter)
 #define bio_iovec(bio)		bio_iter_iovec((bio), (bio)->bi_iter)
 
 #define bio_multiple_segments(bio)				\
@@ -99,20 +93,6 @@ static inline unsigned int bio_cur_bytes(struct bio *bio)
 		return bio->bi_iter.bi_size;
 }
 
-static inline void *bio_data(struct bio *bio)
-{
-	if (bio_has_data(bio))
-		return page_address(bio_page(bio)) + bio_offset(bio);
-
-	return NULL;
-}
-
-#define __bio_kmap_atomic(bio, iter)				\
-	(kmap_atomic(bio_iter_iovec((bio), (iter)).bv_page) +	\
-		bio_iter_iovec((bio), (iter)).bv_offset)
-
-#define __bio_kunmap_atomic(addr)	kunmap_atomic(addr)
-
 static inline struct bio_vec *bio_next_segment(const struct bio *bio,
 					       struct bvec_iter_all *iter)
 {
@@ -238,7 +218,6 @@ struct bio *bio_alloc_bioset(struct block_device *, unsigned,
 
 extern void bio_put(struct bio *);
 
-int bio_add_page(struct bio *, struct page *, unsigned, unsigned);
 void bio_add_virt_nofail(struct bio *, void *, unsigned);
 
 static inline void bio_add_vmalloc(struct bio *bio, void *vaddr, unsigned len)
@@ -265,8 +244,6 @@ extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
 			       struct bio *src, struct bvec_iter *src_iter);
 extern void bio_copy_data(struct bio *dst, struct bio *src);
 
-void bio_free_pages(struct bio *bio);
-
 void zero_fill_bio_iter(struct bio *bio, struct bvec_iter iter);
 
 static inline void zero_fill_bio(struct bio *bio)
@@ -284,30 +261,13 @@ do {						\
 	(dst)->bi_bdev = (src)->bi_bdev;	\
 } while (0)
 
-static inline void *bvec_kmap_irq(struct bio_vec *bvec, unsigned long *flags)
-{
-	return page_address(bvec->bv_page) + bvec->bv_offset;
-}
-
-static inline void bvec_kunmap_irq(char *buffer, unsigned long *flags)
-{
-	*flags = 0;
-}
-
 static inline void *bvec_kmap_local(struct bio_vec *bvec)
 {
-	return page_address(bvec->bv_page) + bvec->bv_offset;
+	return bvec_virt(bvec);
 }
 
 static inline void bvec_kunmap_local(char *buffer) {}
 
-static inline void *__bio_kmap_irq(struct bio *bio, struct bvec_iter iter,
-				   unsigned long *flags)
-{
-	return bvec_kmap_irq(&bio_iter_iovec(bio, iter), flags);
-}
-#define __bio_kunmap_irq(buf, flags)	bvec_kunmap_irq(buf, flags)
-
 #define bio_kmap_irq(bio, flags) \
 	__bio_kmap_irq((bio), (bio)->bi_iter, (flags))
 #define bio_kunmap_irq(buf,flags)	__bio_kunmap_irq(buf, flags)

include/linux/bvec.h

@@ -27,9 +27,8 @@
  * was unsigned short, but we might as well be ready for > 64kB I/O pages
  */
 struct bio_vec {
-	struct page	*bv_page;
+	void		*bv_addr;
 	unsigned int	bv_len;
-	unsigned int	bv_offset;
 };
 
 struct bvec_iter {
@@ -53,21 +52,22 @@ struct bvec_iter_all {
  */
 #define __bvec_iter_bvec(bvec, iter)	(&(bvec)[(iter).bi_idx])
 
-#define bvec_iter_page(bvec, iter)				\
-	(__bvec_iter_bvec((bvec), (iter))->bv_page)
+static inline void *bvec_virt(struct bio_vec *bv)
+{
+	return bv->bv_addr;
+}
+
+#define bvec_iter_addr(bvec, iter)				\
+	(__bvec_iter_bvec((bvec), (iter))->bv_addr + (iter).bi_bvec_done)
 
 #define bvec_iter_len(bvec, iter)				\
 	min((iter).bi_size,					\
 	    __bvec_iter_bvec((bvec), (iter))->bv_len - (iter).bi_bvec_done)
 
-#define bvec_iter_offset(bvec, iter)				\
-	(__bvec_iter_bvec((bvec), (iter))->bv_offset + (iter).bi_bvec_done)
-
 #define bvec_iter_bvec(bvec, iter)				\
 ((struct bio_vec) {						\
-	.bv_page	= bvec_iter_page((bvec), (iter)),	\
+	.bv_addr	= bvec_iter_addr((bvec), (iter)),	\
 	.bv_len		= bvec_iter_len((bvec), (iter)),	\
-	.bv_offset	= bvec_iter_offset((bvec), (iter)),	\
 })
 
 static inline void bvec_iter_advance(const struct bio_vec *bv,

libbcachefs/bcachefs.h

@@ -1037,7 +1037,7 @@ struct bch_fs {
 	struct bio_set		bio_write;
 	struct bio_set		replica_set;
 	struct mutex		bio_bounce_pages_lock;
-	mempool_t		bio_bounce_pages;
+	mempool_t		bio_bounce_bufs;
 	struct bucket_nocow_lock_table
 				nocow_locks;
 	struct rhashtable	promote_table;

libbcachefs/data/checksum.c

@@ -202,15 +202,14 @@ static struct bch_csum __bch2_checksum_bio(struct bch_fs *c, unsigned type,
 
 #ifdef CONFIG_HIGHMEM
 		__bio_for_each_segment(bv, bio, *iter, *iter) {
-			void *p = kmap_local_page(bv.bv_page) + bv.bv_offset;
+			void *p = bvec_kmap_local(&bv);
 
 			bch2_checksum_update(&state, p, bv.bv_len);
 			kunmap_local(p);
 		}
 #else
 		__bio_for_each_bvec(bv, bio, *iter, *iter)
-			bch2_checksum_update(&state, page_address(bv.bv_page) + bv.bv_offset,
-				bv.bv_len);
+			bch2_checksum_update(&state, bvec_virt(&bv), bv.bv_len);
 #endif
 		return (struct bch_csum) { .lo = cpu_to_le64(bch2_checksum_final(&state)) };
 	}
@@ -225,16 +224,14 @@ static struct bch_csum __bch2_checksum_bio(struct bch_fs *c, unsigned type,
 
 #ifdef CONFIG_HIGHMEM
 		__bio_for_each_segment(bv, bio, *iter, *iter) {
-			void *p = kmap_local_page(bv.bv_page) + bv.bv_offset;
+			void *p = bvec_kmap_local(&bv);
 
 			poly1305_update(&dctx, p, bv.bv_len);
 			kunmap_local(p);
 		}
 #else
 		__bio_for_each_bvec(bv, bio, *iter, *iter)
-			poly1305_update(&dctx,
-				page_address(bv.bv_page) + bv.bv_offset,
-				bv.bv_len);
+			poly1305_update(&dctx, bvec_virt(&bv), bv.bv_len);
 #endif
 		poly1305_final(&dctx, digest);
 

libbcachefs/data/compress.c

@@ -95,12 +95,12 @@ static bool bio_phys_contig(struct bio *bio, struct bvec_iter start)
 	void *expected_start = NULL;
 
 	__bio_for_each_bvec(bv, bio, iter, start) {
-		if (expected_start &&
-		    expected_start != page_address(bv.bv_page) + bv.bv_offset)
+		void *bv_addr = bvec_virt(&bv);
+
+		if (expected_start && expected_start != bv_addr)
 			return false;
 
-		expected_start = page_address(bv.bv_page) +
-			bv.bv_offset + bv.bv_len;
+		expected_start = bv_addr + bv.bv_len;
 	}
 
 	return true;
@@ -109,27 +109,27 @@ static bool bio_phys_contig(struct bio *bio, struct bvec_iter start)
 static struct bbuf __bio_map_or_bounce(struct bch_fs *c, struct bio *bio,
 				       struct bvec_iter start, int rw)
 {
-	struct bio_vec bv;
-	struct bvec_iter iter;
-	unsigned nr_pages = 0;
-	struct page *stack_pages[16];
-	struct page **pages = NULL;
-	void *data;
-
 	BUG_ON(start.bi_size > c->opts.encoded_extent_max);
 
-	if (!PageHighMem(bio_iter_page(bio, start)) &&
-	    bio_phys_contig(bio, start))
+#ifndef CONFIG_HIGHMEM
+	if (bio_phys_contig(bio, start))
 		return (struct bbuf) {
 			.c	= c,
-			.b	= page_address(bio_iter_page(bio, start)) +
-				bio_iter_offset(bio, start),
+			.b	= bvec_virt(&bio_iter_iovec(bio, start)),
 			.type	= BB_none,
 			.rw	= rw
 		};
+#endif
 
+#ifdef __KERNEL__
 	/* check if we can map the pages contiguously: */
+	struct bio_vec bv;
+	struct bvec_iter iter;
+	unsigned nr_pages = 0;
+
 	__bio_for_each_segment(bv, bio, iter, start) {
+		BUG_ON(bv.bv_offset + bv.bv_len < PAGE_SIZE);
+
 		if (iter.bi_size != start.bi_size &&
 		    bv.bv_offset)
 			return bio_bounce(c, bio, start, rw);
@@ -143,7 +143,8 @@ static struct bbuf __bio_map_or_bounce(struct bch_fs *c, struct bio *bio,
 
 	BUG_ON(DIV_ROUND_UP(start.bi_size, PAGE_SIZE) > nr_pages);
 
-	pages = nr_pages > ARRAY_SIZE(stack_pages)
+	struct page *stack_pages[16];
+	struct page **pages = nr_pages > ARRAY_SIZE(stack_pages)
 		? kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOFS)
 		: stack_pages;
 	if (!pages)
@@ -153,19 +154,20 @@ static struct bbuf __bio_map_or_bounce(struct bch_fs *c, struct bio *bio,
 	__bio_for_each_segment(bv, bio, iter, start)
 		pages[nr_pages++] = bv.bv_page;
 
-	data = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
+	void *data = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
 	if (pages != stack_pages)
 		kfree(pages);
 
-	if (!data)
-		return bio_bounce(c, bio, start, rw);
+	if (data)
+		return (struct bbuf) {
+			c,
+			data + bio_iter_offset(bio, start),
+			BB_vmap,
+			rw
+		};
+#endif /* __KERNEL__ */
 
-	return (struct bbuf) {
-		c,
-		data + bio_iter_offset(bio, start),
-		BB_vmap,
-		rw
-	};
+	return bio_bounce(c, bio, start, rw);
 }
 
 static struct bbuf bio_map_or_bounce(struct bch_fs *c, struct bio *bio, int rw)

libbcachefs/data/move.c

@@ -452,7 +452,9 @@ root_err:
 			goto retry_root;
 		if (bch2_err_matches(ret, BCH_ERR_data_update_fail))
 			ret = 0; /* failure for this extent, keep going */
-		WARN_ONCE(ret && !bch2_err_matches(ret, EROFS),
+		WARN_ONCE(ret &&
+			  !bch2_err_matches(ret, EROFS) &&
+			  !bch2_err_matches(ret, EIO),
 			  "unhandled error from move_extent: %s", bch2_err_str(ret));
 		return ret;
 	}
@@ -492,7 +494,8 @@ root_err:
 			continue;
 		if (bch2_err_matches(ret, BCH_ERR_data_update_fail))
 			ret = 0; /* failure for this extent, keep going */
-		if (bch2_err_matches(ret, EROFS))
+		if (bch2_err_matches(ret, EROFS) ||
+		    bch2_err_matches(ret, EIO)) /* topology error, btree node read error */
 			break;
 		WARN_ONCE(ret, "unhandled error from move_extent: %s", bch2_err_str(ret));
 next_nondata:
@@ -603,6 +606,7 @@ static int __bch2_move_data_phys(struct moving_context *ctxt,
 		if (bch2_err_matches(ret, BCH_ERR_data_update_fail))
 			ret = 0; /* failure for this extent, keep going */
 		if (bch2_err_matches(ret, EROFS) ||
+		    bch2_err_matches(ret, EIO) ||
 		    bch2_err_matches(ret, BCH_ERR_device_offline))
 			return ret;
 		WARN_ONCE(ret, "unhandled error from move_extent: %s", bch2_err_str(ret));

libbcachefs/data/read.c

@@ -344,7 +344,7 @@ err_remove_hash:
 	BUG_ON(rhashtable_remove_fast(&c->promote_table, &op->hash,
 				      bch_promote_params));
 err:
-	bio_free_pages(&op->write.op.wbio.bio);
+	bch2_bio_free_pages_pool(c, &op->write.op.wbio.bio);
 	/* We may have added to the rhashtable and thus need rcu freeing: */
 	kfree_rcu(op, rcu);
 err_put:
@@ -1253,7 +1253,7 @@ retry_pick:
 						  &c->bio_read_split),
 				 orig);
 
-		bch2_bio_alloc_pages_pool(c, &rbio->bio, sectors << 9);
+		bch2_bio_alloc_pages_pool(c, &rbio->bio, 512, sectors << 9);
 		rbio->bounce	= true;
 	} else if (flags & BCH_READ_must_clone) {
 		/*
@@ -1591,16 +1591,29 @@ void bch2_fs_io_read_exit(struct bch_fs *c)
 		rhashtable_destroy(&c->promote_table);
 	bioset_exit(&c->bio_read_split);
 	bioset_exit(&c->bio_read);
-	mempool_exit(&c->bio_bounce_pages);
+	mempool_exit(&c->bio_bounce_bufs);
+}
+
+static void *bio_bounce_buf_alloc_fn(gfp_t gfp, void *pool_data)
+{
+	return (void *) __get_free_pages(gfp, PAGE_ALLOC_COSTLY_ORDER);
+}
+
+static void bio_bounce_buf_free_fn(void *p, void *pool_data)
+{
+	free_pages((unsigned long) p, PAGE_ALLOC_COSTLY_ORDER);
 }
 
 int bch2_fs_io_read_init(struct bch_fs *c)
 {
-	if (mempool_init_page_pool(&c->bio_bounce_pages,
-				   max_t(unsigned,
-					 c->opts.btree_node_size,
-					 c->opts.encoded_extent_max) /
-				   PAGE_SIZE, 0))
+	if (mempool_init(&c->bio_bounce_bufs,
+			 max_t(unsigned,
+			       c->opts.btree_node_size,
+			       c->opts.encoded_extent_max) /
+			 BIO_BOUNCE_BUF_POOL_LEN,
+			 bio_bounce_buf_alloc_fn,
+			 bio_bounce_buf_free_fn,
+			 NULL))
 		return bch_err_throw(c, ENOMEM_bio_bounce_pages_init);
 
 	if (bioset_init(&c->bio_read, 1, offsetof(struct bch_read_bio, bio),

libbcachefs/data/read.h

@@ -7,6 +7,8 @@
 #include "extents_types.h"
 #include "data/reflink.h"
 
+#define BIO_BOUNCE_BUF_POOL_LEN	(PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)
+
 #ifndef CONFIG_BCACHEFS_NO_LATENCY_ACCT
 void bch2_dev_congested_to_text(struct printbuf *, struct bch_dev *);
 #endif

libbcachefs/data/write.c

@@ -113,43 +113,41 @@ void bch2_latency_acct(struct bch_dev *ca, u64 submit_time, int rw)
 
 void bch2_bio_free_pages_pool(struct bch_fs *c, struct bio *bio)
 {
-	struct bvec_iter_all iter;
-	struct bio_vec *bv;
+	for (struct bio_vec *bv = bio->bi_io_vec;
+	     bv < bio->bi_io_vec + bio->bi_vcnt;
+	     bv++) {
+		void *p = bvec_virt(bv);
 
-	bio_for_each_segment_all(bv, bio, iter)
-		if (bv->bv_page != ZERO_PAGE(0))
-			mempool_free(bv->bv_page, &c->bio_bounce_pages);
+		if (bv->bv_len == BIO_BOUNCE_BUF_POOL_LEN)
+			mempool_free(p, &c->bio_bounce_bufs);
+		else
+			free_pages((unsigned long) p, get_order(bv->bv_len));
+	}
 	bio->bi_vcnt = 0;
 }
 
-static struct page *__bio_alloc_page_pool(struct bch_fs *c, bool *using_mempool)
+static void __bch2_bio_alloc_pages_pool(struct bch_fs *c, struct bio *bio,
+					unsigned bs, size_t size)
 {
-	if (likely(!*using_mempool)) {
-		struct page *page = alloc_page(GFP_NOFS);
-		if (likely(page))
-			return page;
+	mutex_lock(&c->bio_bounce_pages_lock);
 
-		mutex_lock(&c->bio_bounce_pages_lock);
-		*using_mempool = true;
-	}
-	return mempool_alloc(&c->bio_bounce_pages, GFP_NOFS);
+	while (bio->bi_iter.bi_size < size)
+		bio_add_virt_nofail(bio,
+				    mempool_alloc(&c->bio_bounce_bufs, GFP_NOFS),
+				    BIO_BOUNCE_BUF_POOL_LEN);
+
+	bio->bi_iter.bi_size = min(bio->bi_iter.bi_size, size);
+
+	mutex_unlock(&c->bio_bounce_pages_lock);
 }
 
 void bch2_bio_alloc_pages_pool(struct bch_fs *c, struct bio *bio,
-			       size_t size)
+			       unsigned bs, size_t size)
 {
-	bool using_mempool = false;
+	bch2_bio_alloc_pages(bio, c->opts.block_size, size, GFP_NOFS);
 
-	while (size) {
-		struct page *page = __bio_alloc_page_pool(c, &using_mempool);
-		unsigned len = min_t(size_t, PAGE_SIZE, size);
-
-		BUG_ON(!bio_add_page(bio, page, len, 0));
-		size -= len;
-	}
-
-	if (using_mempool)
-		mutex_unlock(&c->bio_bounce_pages_lock);
+	if (bio->bi_iter.bi_size < size)
+		__bch2_bio_alloc_pages_pool(c, bio, bs, size);
 }
 
 /* Extent update path: */
@@ -838,23 +836,22 @@ static struct bio *bch2_write_bio_alloc(struct bch_fs *c,
 		return bio;
 	}
 
-	wbio->bounce		= true;
+	wbio->bounce = true;
+
 
 	/*
 	 * We can't use mempool for more than c->sb.encoded_extent_max
 	 * worth of pages, but we'd like to allocate more if we can:
 	 */
-	bch2_bio_alloc_pages_pool(c, bio,
-				  min_t(unsigned, output_available,
-					c->opts.encoded_extent_max));
+	bch2_bio_alloc_pages(bio,
+			     c->opts.block_size,
+			     output_available,
+			     GFP_NOFS);
 
-	if (bio->bi_iter.bi_size < output_available)
-		*page_alloc_failed =
-			bch2_bio_alloc_pages(bio,
-					     c->opts.block_size,
-					     output_available -
-					     bio->bi_iter.bi_size,
-					     GFP_NOFS) != 0;
+	unsigned required = min(output_available, c->opts.encoded_extent_max);
+
+	if (unlikely(bio->bi_iter.bi_size < required))
+		__bch2_bio_alloc_pages_pool(c, bio, c->opts.block_size, required);
 
 	return bio;
 }

libbcachefs/data/write.h

@@ -9,7 +9,7 @@
 	container_of((_bio), struct bch_write_bio, bio)
 
 void bch2_bio_free_pages_pool(struct bch_fs *, struct bio *);
-void bch2_bio_alloc_pages_pool(struct bch_fs *, struct bio *, size_t);
+void bch2_bio_alloc_pages_pool(struct bch_fs *, struct bio *, unsigned, size_t);
 
 void bch2_submit_wbio_replicas(struct bch_write_bio *, struct bch_fs *,
 			       enum bch_data_type, const struct bkey_i *, bool);

libbcachefs/util/util.c

@@ -606,49 +606,32 @@ void bch2_bio_map(struct bio *bio, void *base, size_t size)
 
 int bch2_bio_alloc_pages(struct bio *bio, unsigned bs, size_t size, gfp_t gfp_mask)
 {
+	BUG_ON(!is_power_of_2(bs));
 	BUG_ON(size & (bs - 1));
-	unsigned bs_pages = DIV_ROUND_UP(bs, PAGE_SIZE);
 
-	/*
-	 * XXX: we could do this by allocating higher order pages, but
-	 *
-	 * - the page allocator gets slower at a certain order (5?) - we'd have
-	 *   to check for this
-	 *
-	 * - bch2_bio_free_pages_pool() probably does not handle compound pages
-	 *   yet
-	 */
-	DARRAY_PREALLOCATED(struct page *, 16) pages;
-	darray_init(&pages);
-	darray_make_room_gfp(&pages, bs_pages, gfp_mask|__GFP_NOFAIL);
+	unsigned max_alloc = max(bs, PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER);
 
-	int ret = 0;
-	while (size) {
-		while (pages.nr < bs_pages) {
-			struct page *page = alloc_pages(gfp_mask, 0);
-			if (!page) {
-				ret = -ENOMEM;
-				goto out;
-			}
+	while (bio->bi_iter.bi_size < size) {
+		unsigned b = min(size - bio->bi_iter.bi_size, max_alloc);
 
-			BUG_ON(darray_push(&pages, page));
-		}
+		BUG_ON(b & (bs - 1));
 
-		while (pages.nr) {
-			BUG_ON(!size);
+#ifdef __KERNEL__
+		/*
+		 * we don't know the device dma alignment, so in kernel make
+		 * sure allocations are page aligned
+		 */
+		void *p = (void *) __get_free_pages(gfp_mask, get_order(b));
+#else
+		void *p = kmalloc(b, gfp_mask);
+#endif
+		if (!p)
+			return -ENOMEM;
 
-			unsigned len = min(PAGE_SIZE, size);
-			size -= len;
-
-			struct page *page = darray_pop(&pages);
-			BUG_ON(!bio_add_page(bio, page, len, 0));
-		}
+		bio_add_virt_nofail(bio, p, b);
 	}
-out:
-	darray_for_each(pages, i)
-		__free_page(*i);
-	darray_exit(&pages);
-	return ret;
+
+	return 0;
 }
 
 u64 bch2_get_random_u64_below(u64 ceil)
@@ -678,9 +661,8 @@ void memcpy_to_bio(struct bio *dst, struct bvec_iter dst_iter, const void *src)
 	struct bvec_iter iter;
 
 	__bio_for_each_segment(bv, dst, iter, dst_iter) {
-		void *dstp = kmap_local_page(bv.bv_page);
-
-		memcpy(dstp + bv.bv_offset, src, bv.bv_len);
+		void *dstp = bvec_kmap_local(&bv);
+		memcpy(dstp, src, bv.bv_len);
 		kunmap_local(dstp);
 
 		src += bv.bv_len;
@@ -693,9 +675,8 @@ void memcpy_from_bio(void *dst, struct bio *src, struct bvec_iter src_iter)
 	struct bvec_iter iter;
 
 	__bio_for_each_segment(bv, src, iter, src_iter) {
-		void *srcp = kmap_local_page(bv.bv_page);
-
-		memcpy(dst, srcp + bv.bv_offset, bv.bv_len);
+		void *srcp = bvec_kmap_local(&bv);
+		memcpy(dst, srcp, bv.bv_len);
 		kunmap_local(srcp);
 
 		dst += bv.bv_len;

linux/bio.c

@@ -64,27 +64,13 @@ const char *blk_status_to_str(blk_status_t status)
 void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
 			struct bio *src, struct bvec_iter *src_iter)
 {
-	struct bio_vec src_bv, dst_bv;
-	void *src_p, *dst_p;
-	unsigned bytes;
-
 	while (src_iter->bi_size && dst_iter->bi_size) {
-		src_bv = bio_iter_iovec(src, *src_iter);
-		dst_bv = bio_iter_iovec(dst, *dst_iter);
+		struct bio_vec src_bv = bio_iter_iovec(src, *src_iter);
+		struct bio_vec dst_bv = bio_iter_iovec(dst, *dst_iter);
 
-		bytes = min(src_bv.bv_len, dst_bv.bv_len);
+		unsigned bytes = min(src_bv.bv_len, dst_bv.bv_len);
 
-		src_p = kmap_atomic(src_bv.bv_page);
-		dst_p = kmap_atomic(dst_bv.bv_page);
-
-		memcpy(dst_p + dst_bv.bv_offset,
-		       src_p + src_bv.bv_offset,
-		       bytes);
-
-		kunmap_atomic(dst_p);
-		kunmap_atomic(src_p);
-
-		flush_dcache_page(dst_bv.bv_page);
+		memcpy(dst_bv.bv_addr, src_bv.bv_addr, bytes);
 
 		bio_advance_iter(src, src_iter, bytes);
 		bio_advance_iter(dst, dst_iter, bytes);
@@ -109,15 +95,11 @@ void bio_copy_data(struct bio *dst, struct bio *src)
 
 void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start)
 {
-	unsigned long flags;
 	struct bio_vec bv;
 	struct bvec_iter iter;
 
-	__bio_for_each_segment(bv, bio, iter, start) {
-		char *data = bvec_kmap_irq(&bv, &flags);
-		memset(data, 0, bv.bv_len);
-		bvec_kunmap_irq(data, &flags);
-	}
+	__bio_for_each_segment(bv, bio, iter, start)
+		memset(bv.bv_addr, 0, bv.bv_len);
 }
 
 static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp)
@@ -165,15 +147,6 @@ struct bio *bio_split(struct bio *bio, int sectors,
 	return split;
 }
 
-void bio_free_pages(struct bio *bio)
-{
-	struct bvec_iter_all iter;
-	struct bio_vec *bvec;
-
-	bio_for_each_segment_all(bvec, bio, iter)
-		__free_page(bvec->bv_page);
-}
-
 void bio_advance(struct bio *bio, unsigned bytes)
 {
 	bio_advance_iter(bio, &bio->bi_iter, bytes);
@@ -208,26 +181,18 @@ void bio_put(struct bio *bio)
 	}
 }
 
-int bio_add_page(struct bio *bio, struct page *page,
-		 unsigned int len, unsigned int off)
+void bio_add_virt_nofail(struct bio *bio, void *vaddr, unsigned len)
 {
 	struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
 
 	WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
 	WARN_ON_ONCE(bio->bi_vcnt >= bio->bi_max_vecs);
 
-	bv->bv_page = page;
-	bv->bv_offset = off;
+	bv->bv_addr = vaddr;
 	bv->bv_len = len;
 
 	bio->bi_iter.bi_size += len;
 	bio->bi_vcnt++;
-	return len;
-}
-
-void bio_add_virt_nofail(struct bio *bio, void *vaddr, unsigned len)
-{
-	bio_add_page(bio, virt_to_page(vaddr), len, offset_in_page(vaddr));
 }
 
 static inline bool bio_remaining_done(struct bio *bio)

linux/blkdev.c

@@ -59,18 +59,15 @@ void generic_make_request(struct bio *bio)
 
 	i = 0;
 	bio_for_each_segment(bv, bio, iter) {
-		void *start = page_address(bv.bv_page) + bv.bv_offset;
-		size_t len = bv.bv_len;
-
 		iov[i++] = (struct iovec) {
-			.iov_base = start,
-			.iov_len = len,
+			.iov_base = bv.bv_addr,
+			.iov_len = bv.bv_len,
 		};
 
 #ifdef CONFIG_VALGRIND
 		/* To be pedantic it should only be on IO completion. */
 		if (bio_op(bio) == REQ_OP_READ)
-			VALGRIND_MAKE_MEM_DEFINED(start, len);
+			VALGRIND_MAKE_MEM_DEFINED(bv.bv_addr, bv.bv_len);
 #endif
 	}