diff --git a/.bcachefs_revision b/.bcachefs_revision index 92bf9ad4..e34b02c1 100644 --- a/.bcachefs_revision +++ b/.bcachefs_revision @@ -1 +1 @@ -02ae70070acc3bc4740d221efa5ff5425cf6fce5 +ae6e8a59d33008f46bb801850840dbd0a7608bbc diff --git a/cmd_migrate.c b/cmd_migrate.c index 1c449554..34348370 100644 --- a/cmd_migrate.c +++ b/cmd_migrate.c @@ -266,7 +266,8 @@ static void write_data(struct bch_fs *c, op.write_point = writepoint_hashed(0); op.pos = POS(dst_inode->bi_inum, dst_offset >> 9); - int ret = bch2_disk_reservation_get(c, &op.res, len >> 9, 0); + int ret = bch2_disk_reservation_get(c, &op.res, len >> 9, + c->opts.data_replicas, 0); if (ret) die("error reserving space in new filesystem: %s", strerror(-ret)); @@ -328,7 +329,7 @@ static void link_data(struct bch_fs *c, struct bch_inode_unpacked *dst, .gen = bucket(ca, b)->mark.gen, }); - ret = bch2_disk_reservation_get(c, &res, sectors, + ret = bch2_disk_reservation_get(c, &res, sectors, 1, BCH_DISK_RESERVATION_NOFAIL); if (ret) die("error reserving space in new filesystem: %s", diff --git a/libbcachefs/alloc.c b/libbcachefs/alloc.c index f7ff8027..f3833846 100644 --- a/libbcachefs/alloc.c +++ b/libbcachefs/alloc.c @@ -1859,6 +1859,27 @@ int bch2_dev_allocator_start(struct bch_dev *ca) return 0; } +static void allocator_start_issue_discards(struct bch_fs *c) +{ + struct bch_dev *ca; + unsigned i, dev_iter; + size_t bu; + + for_each_rw_member(ca, c, dev_iter) { + unsigned done = 0; + + fifo_for_each_entry(bu, &ca->free_inc, i) { + if (done == ca->nr_invalidated) + break; + + blkdev_issue_discard(ca->disk_sb.bdev, + bucket_to_sector(ca, bu), + ca->mi.bucket_size, GFP_NOIO, 0); + done++; + } + } +} + static int __bch2_fs_allocator_start(struct bch_fs *c) { struct bch_dev *ca; @@ -1938,6 +1959,8 @@ static int __bch2_fs_allocator_start(struct bch_fs *c) */ if (invalidating_data) set_bit(BCH_FS_HOLD_BTREE_WRITES, &c->flags); + else + allocator_start_issue_discards(c); /* * XXX: it's possible for this to deadlock waiting on journal reclaim, @@ -1959,12 +1982,12 @@ static int __bch2_fs_allocator_start(struct bch_fs *c) return ret; } + if (invalidating_data) + allocator_start_issue_discards(c); + for_each_rw_member(ca, c, dev_iter) while (ca->nr_invalidated) { BUG_ON(!fifo_pop(&ca->free_inc, bu)); - blkdev_issue_discard(ca->disk_sb.bdev, - bucket_to_sector(ca, bu), - ca->mi.bucket_size, GFP_NOIO, 0); ca->nr_invalidated--; } @@ -1983,7 +2006,7 @@ again: if (btree_node_dirty(b) && (!b->written || b->level)) { rcu_read_unlock(); six_lock_read(&b->lock); - bch2_btree_node_write(c, b, NULL, SIX_LOCK_read); + bch2_btree_node_write(c, b, SIX_LOCK_read); six_unlock_read(&b->lock); goto again; } diff --git a/libbcachefs/btree_cache.c b/libbcachefs/btree_cache.c index 22846d8a..78e36299 100644 --- a/libbcachefs/btree_cache.c +++ b/libbcachefs/btree_cache.c @@ -178,9 +178,9 @@ static int __btree_node_reclaim(struct bch_fs *c, struct btree *b, bool flush) * the post write cleanup: */ if (verify_btree_ondisk(c)) - bch2_btree_node_write(c, b, NULL, SIX_LOCK_intent); + bch2_btree_node_write(c, b, SIX_LOCK_intent); else - __bch2_btree_node_write(c, b, NULL, SIX_LOCK_read); + __bch2_btree_node_write(c, b, SIX_LOCK_read); /* wait for any in flight btree write */ btree_node_wait_on_io(b); @@ -626,7 +626,9 @@ struct btree *bch2_btree_node_get(struct bch_fs *c, struct btree_iter *iter, struct btree *b; struct bset_tree *t; - BUG_ON(level >= BTREE_MAX_DEPTH); + /* btree_node_fill() requires parent to be locked: */ + EBUG_ON(!btree_node_locked(iter, level + 1)); + EBUG_ON(level >= BTREE_MAX_DEPTH); retry: rcu_read_lock(); b = btree_cache_find(bc, k); @@ -763,6 +765,12 @@ struct btree *bch2_btree_node_get_sibling(struct bch_fs *c, if (IS_ERR(ret) && PTR_ERR(ret) == -EINTR) { btree_node_unlock(iter, level); + + if (!bch2_btree_node_relock(iter, level + 1)) { + bch2_btree_iter_set_locks_want(iter, level + 2); + return ERR_PTR(-EINTR); + } + ret = bch2_btree_node_get(c, iter, &tmp.k, level, SIX_LOCK_intent); } diff --git a/libbcachefs/btree_gc.c b/libbcachefs/btree_gc.c index 9f1071e5..43e64d27 100644 --- a/libbcachefs/btree_gc.c +++ b/libbcachefs/btree_gc.c @@ -734,7 +734,7 @@ static void bch2_coalesce_nodes(struct bch_fs *c, struct btree_iter *iter, bch2_btree_build_aux_trees(n); six_unlock_write(&n->lock); - bch2_btree_node_write(c, n, &as->cl, SIX_LOCK_intent); + bch2_btree_node_write(c, n, SIX_LOCK_intent); } /* diff --git a/libbcachefs/btree_io.c b/libbcachefs/btree_io.c index 3f87e91e..fb76b29e 100644 --- a/libbcachefs/btree_io.c +++ b/libbcachefs/btree_io.c @@ -1425,6 +1425,19 @@ err: void bch2_btree_complete_write(struct bch_fs *c, struct btree *b, struct btree_write *w) { + unsigned long old, new, v = READ_ONCE(b->will_make_reachable); + + do { + old = new = v; + if (!(old & 1)) + break; + + new &= ~1UL; + } while ((v = cmpxchg(&b->will_make_reachable, old, new)) != old); + + if (old & 1) + closure_put(&((struct btree_update *) new)->cl); + bch2_journal_pin_drop(&c->journal, &w->journal); closure_wake_up(&w->wait); } @@ -1441,7 +1454,6 @@ static void bch2_btree_node_write_error(struct bch_fs *c, struct btree_write_bio *wbio) { struct btree *b = wbio->wbio.bio.bi_private; - struct closure *cl = wbio->cl; __BKEY_PADDED(k, BKEY_BTREE_PTR_VAL_U64s_MAX) tmp; struct bkey_i_extent *new_key; struct bkey_s_extent e; @@ -1488,8 +1500,6 @@ out: bch2_btree_iter_unlock(&iter); bio_put(&wbio->wbio.bio); btree_node_write_done(c, b); - if (cl) - closure_put(cl); return; err: set_btree_node_noevict(b); @@ -1520,7 +1530,6 @@ static void btree_node_write_work(struct work_struct *work) { struct btree_write_bio *wbio = container_of(work, struct btree_write_bio, work); - struct closure *cl = wbio->cl; struct bch_fs *c = wbio->wbio.c; struct btree *b = wbio->wbio.bio.bi_private; @@ -1542,8 +1551,6 @@ static void btree_node_write_work(struct work_struct *work) bio_put(&wbio->wbio.bio); btree_node_write_done(c, b); - if (cl) - closure_put(cl); } static void btree_node_write_endio(struct bio *bio) @@ -1598,7 +1605,6 @@ static int validate_bset_for_write(struct bch_fs *c, struct btree *b, } void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, - struct closure *parent, enum six_lock_type lock_type_held) { struct btree_write_bio *wbio; @@ -1651,7 +1657,7 @@ void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, BUG_ON(btree_node_fake(b)); BUG_ON(!list_empty(&b->write_blocked)); - BUG_ON((b->will_make_reachable != NULL) != !b->written); + BUG_ON((b->will_make_reachable != 0) != !b->written); BUG_ON(b->written >= c->opts.btree_node_size); BUG_ON(bset_written(b, btree_bset_last(b))); @@ -1786,7 +1792,6 @@ void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, struct btree_write_bio, wbio.bio); wbio_init(&wbio->wbio.bio); wbio->data = data; - wbio->cl = parent; wbio->wbio.order = order; wbio->wbio.used_mempool = used_mempool; wbio->wbio.bio.bi_opf = REQ_OP_WRITE|REQ_META|REQ_FUA; @@ -1794,9 +1799,6 @@ void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, wbio->wbio.bio.bi_end_io = btree_node_write_endio; wbio->wbio.bio.bi_private = b; - if (parent) - closure_get(parent); - bch2_bio_map(&wbio->wbio.bio, data); /* @@ -1893,7 +1895,6 @@ bool bch2_btree_post_write_cleanup(struct bch_fs *c, struct btree *b) * Use this one if the node is intent locked: */ void bch2_btree_node_write(struct bch_fs *c, struct btree *b, - struct closure *parent, enum six_lock_type lock_type_held) { BUG_ON(lock_type_held == SIX_LOCK_write); @@ -1901,7 +1902,7 @@ void bch2_btree_node_write(struct bch_fs *c, struct btree *b, if (lock_type_held == SIX_LOCK_intent || six_trylock_convert(&b->lock, SIX_LOCK_read, SIX_LOCK_intent)) { - __bch2_btree_node_write(c, b, parent, SIX_LOCK_intent); + __bch2_btree_node_write(c, b, SIX_LOCK_intent); /* don't cycle lock unnecessarily: */ if (btree_node_just_written(b)) { @@ -1913,7 +1914,7 @@ void bch2_btree_node_write(struct bch_fs *c, struct btree *b, if (lock_type_held == SIX_LOCK_read) six_lock_downgrade(&b->lock); } else { - __bch2_btree_node_write(c, b, parent, SIX_LOCK_read); + __bch2_btree_node_write(c, b, SIX_LOCK_read); } } diff --git a/libbcachefs/btree_io.h b/libbcachefs/btree_io.h index c8417ac3..66053d53 100644 --- a/libbcachefs/btree_io.h +++ b/libbcachefs/btree_io.h @@ -19,7 +19,6 @@ struct btree_read_bio { }; struct btree_write_bio { - struct closure *cl; void *data; struct work_struct work; struct bch_write_bio wbio; @@ -91,22 +90,41 @@ void bch2_btree_complete_write(struct bch_fs *, struct btree *, void bch2_btree_write_error_work(struct work_struct *); void __bch2_btree_node_write(struct bch_fs *, struct btree *, - struct closure *, enum six_lock_type); + enum six_lock_type); bool bch2_btree_post_write_cleanup(struct bch_fs *, struct btree *); void bch2_btree_node_write(struct bch_fs *, struct btree *, - struct closure *, enum six_lock_type); + enum six_lock_type); -#define bch2_btree_node_write_dirty(_c, _b, _cl, cond) \ +/* + * btree_node_dirty() can be cleared with only a read lock, + * and for bch2_btree_node_write_cond() we want to set need_write iff it's + * still dirty: + */ +static inline void set_btree_node_need_write_if_dirty(struct btree *b) +{ + unsigned long old, new, v = READ_ONCE(b->flags); + + do { + old = new = v; + + if (!(old & (1 << BTREE_NODE_dirty))) + return; + + new |= (1 << BTREE_NODE_need_write); + } while ((v = cmpxchg(&b->flags, old, new)) != old); +} + +#define bch2_btree_node_write_cond(_c, _b, cond) \ do { \ while ((_b)->written && btree_node_dirty(_b) && (cond)) { \ - set_btree_node_need_write(_b); \ - \ - if (!btree_node_may_write(_b)) \ + if (!btree_node_may_write(_b)) { \ + set_btree_node_need_write_if_dirty(_b); \ break; \ + } \ \ if (!btree_node_write_in_flight(_b)) { \ - bch2_btree_node_write(_c, _b, _cl, SIX_LOCK_read);\ + bch2_btree_node_write(_c, _b, SIX_LOCK_read); \ break; \ } \ \ diff --git a/libbcachefs/btree_types.h b/libbcachefs/btree_types.h index fb2f7e21..e86c6bce 100644 --- a/libbcachefs/btree_types.h +++ b/libbcachefs/btree_types.h @@ -126,7 +126,7 @@ struct btree { * another write - because that write also won't yet be reachable and * marking it as completed before it's reachable would be incorrect: */ - struct btree_update *will_make_reachable; + unsigned long will_make_reachable; struct btree_ob_ref ob; diff --git a/libbcachefs/btree_update_interior.c b/libbcachefs/btree_update_interior.c index a0f37c4c..e92a6a3a 100644 --- a/libbcachefs/btree_update_interior.c +++ b/libbcachefs/btree_update_interior.c @@ -270,6 +270,17 @@ void bch2_btree_node_free_never_inserted(struct bch_fs *c, struct btree *b) void bch2_btree_node_free_inmem(struct bch_fs *c, struct btree *b, struct btree_iter *iter) { + /* + * Is this a node that isn't reachable on disk yet? + * + * Nodes that aren't reachable yet have writes blocked until they're + * reachable - now that we've cancelled any pending writes and moved + * things waiting on that write to wait on this update, we can drop this + * node from the list of nodes that the other update is making + * reachable, prior to freeing it: + */ + btree_update_drop_new_node(c, b); + bch2_btree_iter_node_drop_linked(iter, b); __btree_node_free(c, b, iter); @@ -503,8 +514,7 @@ static struct btree_reserve *bch2_btree_reserve_get(struct bch_fs *c, struct btree *b; struct disk_reservation disk_res = { 0, 0 }; unsigned sectors = nr_nodes * c->opts.btree_node_size; - int ret, disk_res_flags = BCH_DISK_RESERVATION_GC_LOCK_HELD| - BCH_DISK_RESERVATION_METADATA; + int ret, disk_res_flags = BCH_DISK_RESERVATION_GC_LOCK_HELD; if (flags & BTREE_INSERT_NOFAIL) disk_res_flags |= BCH_DISK_RESERVATION_NOFAIL; @@ -517,7 +527,9 @@ static struct btree_reserve *bch2_btree_reserve_get(struct bch_fs *c, if (ret) return ERR_PTR(ret); - if (bch2_disk_reservation_get(c, &disk_res, sectors, disk_res_flags)) + if (bch2_disk_reservation_get(c, &disk_res, sectors, + c->opts.metadata_replicas, + disk_res_flags)) return ERR_PTR(-ENOSPC); BUG_ON(nr_nodes > BTREE_RESERVE_MAX); @@ -597,12 +609,17 @@ static void btree_update_nodes_reachable(struct closure *cl) while (as->nr_new_nodes) { struct btree *b = as->new_nodes[--as->nr_new_nodes]; - BUG_ON(b->will_make_reachable != as); - b->will_make_reachable = NULL; + BUG_ON(b->will_make_reachable && + (struct btree_update *) b->will_make_reachable != as); + b->will_make_reachable = 0; mutex_unlock(&c->btree_interior_update_lock); + /* + * b->will_make_reachable prevented it from being written, so + * write it now if it needs to be written: + */ six_lock_read(&b->lock); - bch2_btree_node_write_dirty(c, b, NULL, btree_node_need_write(b)); + bch2_btree_node_write_cond(c, b, btree_node_need_write(b)); six_unlock_read(&b->lock); mutex_lock(&c->btree_interior_update_lock); } @@ -651,8 +668,11 @@ retry: list_del(&as->write_blocked_list); mutex_unlock(&c->btree_interior_update_lock); - bch2_btree_node_write_dirty(c, b, NULL, - btree_node_need_write(b)); + /* + * b->write_blocked prevented it from being written, so + * write it now if it needs to be written: + */ + bch2_btree_node_write_cond(c, b, btree_node_need_write(b)); six_unlock_read(&b->lock); break; @@ -851,17 +871,25 @@ static void btree_node_will_make_reachable(struct btree_update *as, BUG_ON(b->will_make_reachable); as->new_nodes[as->nr_new_nodes++] = b; - b->will_make_reachable = as; + b->will_make_reachable = 1UL|(unsigned long) as; + + closure_get(&as->cl); mutex_unlock(&c->btree_interior_update_lock); } -static void __btree_interior_update_drop_new_node(struct btree *b) +static void btree_update_drop_new_node(struct bch_fs *c, struct btree *b) { - struct btree_update *as = b->will_make_reachable; + struct btree_update *as; + unsigned long v; unsigned i; - BUG_ON(!as); + if (!b->will_make_reachable) + return; + mutex_lock(&c->btree_interior_update_lock); + v = xchg(&b->will_make_reachable, 0); + + as = (struct btree_update *) (v & ~1UL); for (i = 0; i < as->nr_new_nodes; i++) if (as->new_nodes[i] == b) goto found; @@ -869,14 +897,10 @@ static void __btree_interior_update_drop_new_node(struct btree *b) BUG(); found: array_remove_item(as->new_nodes, as->nr_new_nodes, i); - b->will_make_reachable = NULL; -} - -static void btree_update_drop_new_node(struct bch_fs *c, struct btree *b) -{ - mutex_lock(&c->btree_interior_update_lock); - __btree_interior_update_drop_new_node(b); mutex_unlock(&c->btree_interior_update_lock); + + if (v & 1) + closure_put(&as->cl); } static void btree_interior_update_add_node_reference(struct btree_update *as, @@ -952,6 +976,12 @@ void bch2_btree_interior_update_will_free_node(struct btree_update *as, clear_btree_node_need_write(b); w = btree_current_write(b); + /* + * Does this node have any btree_update operations waiting on this node + * to be written? + * + * If so, wake them up when this btree_update operation is reachable: + */ llist_for_each_entry_safe(cl, cl_n, llist_del_all(&w->wait.list), list) llist_add(&cl->list, &as->wait.list); @@ -972,9 +1002,6 @@ void bch2_btree_interior_update_will_free_node(struct btree_update *as, &as->journal, interior_update_flush); bch2_journal_pin_drop(&c->journal, &w->journal); - if (b->will_make_reachable) - __btree_interior_update_drop_new_node(b); - mutex_unlock(&c->btree_interior_update_lock); } @@ -1338,7 +1365,7 @@ static void btree_split(struct btree_update *as, struct btree *b, six_unlock_write(&n2->lock); six_unlock_write(&n1->lock); - bch2_btree_node_write(c, n2, &as->cl, SIX_LOCK_intent); + bch2_btree_node_write(c, n2, SIX_LOCK_intent); /* * Note that on recursive parent_keys == keys, so we @@ -1356,7 +1383,8 @@ static void btree_split(struct btree_update *as, struct btree *b, n3->sib_u64s[1] = U16_MAX; btree_split_insert_keys(as, n3, iter, &as->parent_keys); - bch2_btree_node_write(c, n3, &as->cl, SIX_LOCK_intent); + + bch2_btree_node_write(c, n3, SIX_LOCK_intent); } } else { trace_btree_compact(c, b); @@ -1367,7 +1395,7 @@ static void btree_split(struct btree_update *as, struct btree *b, bch2_keylist_add(&as->parent_keys, &n1->key); } - bch2_btree_node_write(c, n1, &as->cl, SIX_LOCK_intent); + bch2_btree_node_write(c, n1, SIX_LOCK_intent); /* New nodes all written, now make them visible: */ @@ -1668,7 +1696,7 @@ retry: bch2_keylist_add(&as->parent_keys, &delete); bch2_keylist_add(&as->parent_keys, &n->key); - bch2_btree_node_write(c, n, &as->cl, SIX_LOCK_intent); + bch2_btree_node_write(c, n, SIX_LOCK_intent); bch2_btree_insert_node(as, parent, iter, &as->parent_keys); @@ -1726,7 +1754,7 @@ static int __btree_node_rewrite(struct bch_fs *c, struct btree_iter *iter, trace_btree_gc_rewrite_node(c, b); - bch2_btree_node_write(c, n, &as->cl, SIX_LOCK_intent); + bch2_btree_node_write(c, n, SIX_LOCK_intent); if (parent) { bch2_btree_insert_node(as, parent, iter, diff --git a/libbcachefs/btree_update_leaf.c b/libbcachefs/btree_update_leaf.c index e62e0d2e..dbf70569 100644 --- a/libbcachefs/btree_update_leaf.c +++ b/libbcachefs/btree_update_leaf.c @@ -109,7 +109,7 @@ static void __btree_node_flush(struct journal *j, struct journal_entry_pin *pin, struct btree *b = container_of(w, struct btree, writes[i]); six_lock_read(&b->lock); - bch2_btree_node_write_dirty(c, b, NULL, + bch2_btree_node_write_cond(c, b, (btree_current_write(b) == w && w->journal.pin_list == journal_seq_pin(j, seq))); six_unlock_read(&b->lock); diff --git a/libbcachefs/buckets.c b/libbcachefs/buckets.c index 43133cbb..4ea89a97 100644 --- a/libbcachefs/buckets.c +++ b/libbcachefs/buckets.c @@ -713,8 +713,6 @@ int bch2_disk_reservation_add(struct bch_fs *c, struct disk_reservation *res, s64 sectors_available; int ret; - sectors *= res->nr_replicas; - lg_local_lock(&c->usage_lock); stats = this_cpu_ptr(c->usage_percpu); @@ -788,19 +786,6 @@ recalculate: return ret; } -int bch2_disk_reservation_get(struct bch_fs *c, - struct disk_reservation *res, - unsigned sectors, int flags) -{ - res->sectors = 0; - res->gen = c->capacity_gen; - res->nr_replicas = (flags & BCH_DISK_RESERVATION_METADATA) - ? c->opts.metadata_replicas - : c->opts.data_replicas; - - return bch2_disk_reservation_add(c, res, sectors, flags); -} - /* Startup/shutdown: */ static void buckets_free_rcu(struct rcu_head *rcu) diff --git a/libbcachefs/buckets.h b/libbcachefs/buckets.h index 86e72829..fda7fd70 100644 --- a/libbcachefs/buckets.h +++ b/libbcachefs/buckets.h @@ -230,16 +230,36 @@ static inline void bch2_disk_reservation_put(struct bch_fs *c, } #define BCH_DISK_RESERVATION_NOFAIL (1 << 0) -#define BCH_DISK_RESERVATION_METADATA (1 << 1) -#define BCH_DISK_RESERVATION_GC_LOCK_HELD (1 << 2) -#define BCH_DISK_RESERVATION_BTREE_LOCKS_HELD (1 << 3) +#define BCH_DISK_RESERVATION_GC_LOCK_HELD (1 << 1) +#define BCH_DISK_RESERVATION_BTREE_LOCKS_HELD (1 << 2) int bch2_disk_reservation_add(struct bch_fs *, struct disk_reservation *, unsigned, int); -int bch2_disk_reservation_get(struct bch_fs *, - struct disk_reservation *, - unsigned, int); + +static inline struct disk_reservation +bch2_disk_reservation_init(struct bch_fs *c, unsigned nr_replicas) +{ + return (struct disk_reservation) { + .sectors = 0, +#if 0 + /* not used yet: */ + .gen = c->capacity_gen, +#endif + .nr_replicas = nr_replicas, + }; +} + +static inline int bch2_disk_reservation_get(struct bch_fs *c, + struct disk_reservation *res, + unsigned sectors, + unsigned nr_replicas, + int flags) +{ + *res = bch2_disk_reservation_init(c, nr_replicas); + + return bch2_disk_reservation_add(c, res, sectors * nr_replicas, flags); +} int bch2_dev_buckets_resize(struct bch_fs *, struct bch_dev *, u64); void bch2_dev_buckets_free(struct bch_dev *); diff --git a/libbcachefs/extents.c b/libbcachefs/extents.c index bceea486..7e7c38ef 100644 --- a/libbcachefs/extents.c +++ b/libbcachefs/extents.c @@ -1264,7 +1264,8 @@ extent_insert_check_split_compressed(struct extent_insert_state *s, switch (bch2_disk_reservation_add(c, s->trans->disk_res, - sectors, flags)) { + sectors * bch2_extent_nr_dirty_ptrs(k), + flags)) { case 0: break; case -ENOSPC: diff --git a/libbcachefs/fs-io.c b/libbcachefs/fs-io.c index 66374a9c..7693520d 100644 --- a/libbcachefs/fs-io.c +++ b/libbcachefs/fs-io.c @@ -27,9 +27,14 @@ #include #include +struct quota_res { + u64 sectors; +}; + struct i_sectors_hook { struct extent_insert_hook hook; struct bch_inode_info *inode; + struct quota_res quota_res; s64 sectors; u64 new_i_size; unsigned flags; @@ -49,6 +54,7 @@ struct bchfs_write_op { struct bch_writepage_io { struct closure cl; + u64 new_sectors; /* must be last: */ struct bchfs_write_op op; @@ -61,6 +67,7 @@ struct dio_write { unsigned loop:1, sync:1, free_iov:1; + struct quota_res quota_res; struct iov_iter iter; struct iovec inline_vecs[2]; @@ -106,6 +113,67 @@ static int write_invalidate_inode_pages_range(struct address_space *mapping, return ret; } +/* quotas */ + +#ifdef CONFIG_BCACHEFS_QUOTA + +static void bch2_quota_reservation_put(struct bch_fs *c, + struct bch_inode_info *inode, + struct quota_res *res) +{ + if (!res->sectors) + return; + + mutex_lock(&inode->ei_update_lock); + BUG_ON(res->sectors > inode->ei_quota_reserved); + + bch2_quota_acct(c, inode->ei_qid, Q_SPC, + -((s64) res->sectors), BCH_QUOTA_PREALLOC); + inode->ei_quota_reserved -= res->sectors; + mutex_unlock(&inode->ei_update_lock); + + res->sectors = 0; +} + +static int bch2_quota_reservation_add(struct bch_fs *c, + struct bch_inode_info *inode, + struct quota_res *res, + unsigned sectors, + bool check_enospc) +{ + int ret; + + mutex_lock(&inode->ei_update_lock); + ret = bch2_quota_acct(c, inode->ei_qid, Q_SPC, sectors, + check_enospc ? BCH_QUOTA_PREALLOC : BCH_QUOTA_NOCHECK); + if (likely(!ret)) { + inode->ei_quota_reserved += sectors; + res->sectors += sectors; + } + mutex_unlock(&inode->ei_update_lock); + + return ret; +} + +#else + +static void bch2_quota_reservation_put(struct bch_fs *c, + struct bch_inode_info *inode, + struct quota_res *res) +{ +} + +static int bch2_quota_reservation_add(struct bch_fs *c, + struct bch_inode_info *inode, + struct quota_res *res, + unsigned sectors, + bool check_enospc) +{ + return 0; +} + +#endif + /* i_size updates: */ static int inode_set_size(struct bch_inode_info *inode, @@ -127,16 +195,28 @@ static int __must_check bch2_write_inode_size(struct bch_fs *c, return __bch2_write_inode(c, inode, inode_set_size, &new_size); } -static void __i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode, int sectors) +static void __i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode, + struct quota_res *quota_res, int sectors) { +#ifdef CONFIG_BCACHEFS_QUOTA + if (quota_res && sectors > 0) { + BUG_ON(sectors > quota_res->sectors); + BUG_ON(sectors > inode->ei_quota_reserved); + + quota_res->sectors -= sectors; + inode->ei_quota_reserved -= sectors; + } else { + bch2_quota_acct(c, inode->ei_qid, Q_SPC, sectors, BCH_QUOTA_WARN); + } +#endif inode->v.i_blocks += sectors; - bch2_quota_acct(c, inode->ei_qid, Q_SPC, sectors, BCH_QUOTA_WARN); } -static void i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode, int sectors) +static void i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode, + struct quota_res *quota_res, int sectors) { mutex_lock(&inode->ei_update_lock); - __i_sectors_acct(c, inode, sectors); + __i_sectors_acct(c, inode, quota_res, sectors); mutex_unlock(&inode->ei_update_lock); } @@ -185,11 +265,13 @@ static int i_sectors_dirty_finish(struct bch_fs *c, struct i_sectors_hook *h) if (h->new_i_size != U64_MAX) i_size_write(&h->inode->v, h->new_i_size); - __i_sectors_acct(c, h->inode, h->sectors); + __i_sectors_acct(c, h->inode, &h->quota_res, h->sectors); ret = __bch2_write_inode(c, h->inode, i_sectors_dirty_finish_fn, h); mutex_unlock(&h->inode->ei_update_lock); + bch2_quota_reservation_put(c, h->inode, &h->quota_res); + h->sectors = 0; return ret; @@ -400,9 +482,12 @@ err: bch2_btree_iter_unlock(&extent_iter); bch2_btree_iter_unlock(&inode_iter); - if (op->is_dio) - i_sectors_acct(wop->c, op->inode, + if (op->is_dio) { + struct dio_write *dio = container_of(op, struct dio_write, iop); + + i_sectors_acct(wop->c, op->inode, &dio->quota_res, op->sectors_added - orig_sectors_added); + } return ret; } @@ -446,23 +531,23 @@ static inline struct bch_io_opts io_opts(struct bch_fs *c, struct bch_inode_info */ struct bch_page_state { union { struct { - /* - * page is _fully_ written on disk, and not compressed - which means to - * write this page we don't have to reserve space (the new write will - * never take up more space on disk than what it's overwriting) - */ - unsigned allocated:1; + /* existing data: */ + unsigned sectors:PAGE_SECTOR_SHIFT + 1; + unsigned nr_replicas:4; + unsigned compressed:1; /* Owns PAGE_SECTORS sized reservation: */ unsigned reserved:1; - unsigned nr_replicas:4; + unsigned reservation_replicas:4; + + /* Owns PAGE_SECTORS sized quota reservation: */ + unsigned quota_reserved:1; /* * Number of sectors on disk - for i_blocks * Uncompressed size, not compressed size: */ - u8 sectors; - u8 dirty_sectors; + unsigned dirty_sectors:PAGE_SECTOR_SHIFT + 1; }; /* for cmpxchg: */ unsigned long v; @@ -497,54 +582,95 @@ static inline struct bch_page_state *page_state(struct page *page) return s; } -static void bch2_put_page_reservation(struct bch_fs *c, struct page *page) +static inline unsigned page_res_sectors(struct bch_page_state s) { - struct disk_reservation res = { .sectors = PAGE_SECTORS }; - struct bch_page_state s; - s = page_state_cmpxchg(page_state(page), s, { - if (!s.reserved) - return; - s.reserved = 0; - }); + return s.reserved ? s.reservation_replicas * PAGE_SECTORS : 0; +} +static void __bch2_put_page_reservation(struct bch_fs *c, struct bch_inode_info *inode, + struct bch_page_state s) +{ + struct disk_reservation res = { .sectors = page_res_sectors(s) }; + struct quota_res quota_res = { .sectors = s.quota_reserved ? PAGE_SECTORS : 0 }; + + bch2_quota_reservation_put(c, inode, "a_res); bch2_disk_reservation_put(c, &res); } -static int bch2_get_page_reservation(struct bch_fs *c, struct page *page, - bool check_enospc) +static void bch2_put_page_reservation(struct bch_fs *c, struct bch_inode_info *inode, + struct page *page) { - struct bch_page_state *s = page_state(page), new; - struct disk_reservation res; - int ret = 0; + struct bch_page_state s; - BUG_ON(s->allocated && s->sectors != PAGE_SECTORS); - - if (s->allocated || s->reserved) - return 0; - - ret = bch2_disk_reservation_get(c, &res, PAGE_SECTORS, !check_enospc - ? BCH_DISK_RESERVATION_NOFAIL : 0); - if (ret) - return ret; - - page_state_cmpxchg(s, new, { - if (new.reserved) { - bch2_disk_reservation_put(c, &res); - return 0; - } - new.reserved = 1; - new.nr_replicas = res.nr_replicas; + s = page_state_cmpxchg(page_state(page), s, { + s.reserved = 0; + s.quota_reserved = 0; }); - return 0; + __bch2_put_page_reservation(c, inode, s); +} + +static int bch2_get_page_reservation(struct bch_fs *c, struct bch_inode_info *inode, + struct page *page, bool check_enospc) +{ + struct bch_page_state *s = page_state(page), new, old; + struct disk_reservation disk_res = bch2_disk_reservation_init(c, + READ_ONCE(c->opts.data_replicas)); + struct quota_res quota_res = { 0 }; + int ret = 0; + + /* + * XXX: this could likely be quite a bit simpler, page reservations + * _should_ only be manipulated with page locked: + */ + + old = page_state_cmpxchg(s, new, { + if (new.reserved + ? (new.reservation_replicas < disk_res.nr_replicas) + : (new.sectors < PAGE_SECTORS || + new.nr_replicas < disk_res.nr_replicas || + new.compressed)) { + int sectors = (disk_res.nr_replicas * PAGE_SECTORS - + page_res_sectors(new) - + disk_res.sectors); + + if (sectors > 0) { + ret = bch2_disk_reservation_add(c, &disk_res, sectors, + !check_enospc + ? BCH_DISK_RESERVATION_NOFAIL : 0); + if (unlikely(ret)) + goto err; + } + + new.reserved = 1; + new.reservation_replicas = disk_res.nr_replicas; + } + + if (!new.quota_reserved && + new.sectors + new.dirty_sectors < PAGE_SECTORS) { + ret = bch2_quota_reservation_add(c, inode, "a_res, + PAGE_SECTORS - quota_res.sectors, + check_enospc); + if (unlikely(ret)) + goto err; + + new.quota_reserved = 1; + } + }); + + quota_res.sectors -= (new.quota_reserved - old.quota_reserved) * PAGE_SECTORS; + disk_res.sectors -= page_res_sectors(new) - page_res_sectors(old); +err: + bch2_quota_reservation_put(c, inode, "a_res); + bch2_disk_reservation_put(c, &disk_res); + return ret; } static void bch2_clear_page_bits(struct page *page) { struct bch_inode_info *inode = to_bch_ei(page->mapping->host); struct bch_fs *c = inode->v.i_sb->s_fs_info; - struct disk_reservation res = { .sectors = PAGE_SECTORS }; struct bch_page_state s; if (!PagePrivate(page)) @@ -554,28 +680,122 @@ static void bch2_clear_page_bits(struct page *page) ClearPagePrivate(page); if (s.dirty_sectors) - i_sectors_acct(c, inode, -s.dirty_sectors); + i_sectors_acct(c, inode, NULL, -s.dirty_sectors); - if (s.reserved) - bch2_disk_reservation_put(c, &res); + __bch2_put_page_reservation(c, inode, s); } int bch2_set_page_dirty(struct page *page) { struct bch_inode_info *inode = to_bch_ei(page->mapping->host); struct bch_fs *c = inode->v.i_sb->s_fs_info; + struct quota_res quota_res = { 0 }; struct bch_page_state old, new; old = page_state_cmpxchg(page_state(page), new, new.dirty_sectors = PAGE_SECTORS - new.sectors; + new.quota_reserved = 0; ); + quota_res.sectors += old.quota_reserved * PAGE_SECTORS; + if (old.dirty_sectors != new.dirty_sectors) - i_sectors_acct(c, inode, new.dirty_sectors - old.dirty_sectors); + i_sectors_acct(c, inode, "a_res, + new.dirty_sectors - old.dirty_sectors); + bch2_quota_reservation_put(c, inode, "a_res); return __set_page_dirty_nobuffers(page); } +int bch2_page_mkwrite(struct vm_fault *vmf) +{ + struct page *page = vmf->page; + struct file *file = vmf->vma->vm_file; + struct bch_inode_info *inode = file_bch_inode(file); + struct address_space *mapping = inode->v.i_mapping; + struct bch_fs *c = inode->v.i_sb->s_fs_info; + int ret = VM_FAULT_LOCKED; + + sb_start_pagefault(inode->v.i_sb); + file_update_time(file); + + /* + * Not strictly necessary, but helps avoid dio writes livelocking in + * write_invalidate_inode_pages_range() - can drop this if/when we get + * a write_invalidate_inode_pages_range() that works without dropping + * page lock before invalidating page + */ + if (current->pagecache_lock != &mapping->add_lock) + pagecache_add_get(&mapping->add_lock); + + lock_page(page); + if (page->mapping != mapping || + page_offset(page) > i_size_read(&inode->v)) { + unlock_page(page); + ret = VM_FAULT_NOPAGE; + goto out; + } + + if (bch2_get_page_reservation(c, inode, page, true)) { + unlock_page(page); + ret = VM_FAULT_SIGBUS; + goto out; + } + + if (!PageDirty(page)) + set_page_dirty(page); + wait_for_stable_page(page); +out: + if (current->pagecache_lock != &mapping->add_lock) + pagecache_add_put(&mapping->add_lock); + sb_end_pagefault(inode->v.i_sb); + return ret; +} + +void bch2_invalidatepage(struct page *page, unsigned int offset, + unsigned int length) +{ + EBUG_ON(!PageLocked(page)); + EBUG_ON(PageWriteback(page)); + + if (offset || length < PAGE_SIZE) + return; + + bch2_clear_page_bits(page); +} + +int bch2_releasepage(struct page *page, gfp_t gfp_mask) +{ + EBUG_ON(!PageLocked(page)); + EBUG_ON(PageWriteback(page)); + + if (PageDirty(page)) + return 0; + + bch2_clear_page_bits(page); + return 1; +} + +#ifdef CONFIG_MIGRATION +int bch2_migrate_page(struct address_space *mapping, struct page *newpage, + struct page *page, enum migrate_mode mode) +{ + int ret; + + ret = migrate_page_move_mapping(mapping, newpage, page, NULL, mode, 0); + if (ret != MIGRATEPAGE_SUCCESS) + return ret; + + if (PagePrivate(page)) { + *page_state(newpage) = *page_state(page); + ClearPagePrivate(page); + } + + migrate_page_copy(newpage, page); + return MIGRATEPAGE_SUCCESS; +} +#endif + /* readpages/writepages: */ static bool bio_can_add_page_contig(struct bio *bio, struct page *page) @@ -601,7 +821,7 @@ static int bio_add_page_contig(struct bio *bio, struct page *page) { sector_t offset = (sector_t) page->index << PAGE_SECTOR_SHIFT; - BUG_ON(!bio->bi_max_vecs); + EBUG_ON(!bio->bi_max_vecs); if (!bio->bi_vcnt) bio->bi_iter.bi_sector = offset; @@ -612,6 +832,8 @@ static int bio_add_page_contig(struct bio *bio, struct page *page) return 0; } +/* readpage(s): */ + static void bch2_readpages_end_io(struct bio *bio) { struct bio_vec *bv; @@ -638,16 +860,22 @@ struct readpages_iter { unsigned nr_pages; }; -static int readpage_add_page(struct readpages_iter *iter, struct page *page) +static inline void page_state_init_for_read(struct page *page) { struct bch_page_state *s = page_state(page); - int ret; BUG_ON(s->reserved); - s->allocated = 1; - s->sectors = 0; + s->sectors = 0; + s->compressed = 0; +} + +static int readpage_add_page(struct readpages_iter *iter, struct page *page) +{ + int ret; prefetchw(&page->flags); + page_state_init_for_read(page); + ret = add_to_page_cache_lru(page, iter->mapping, page->index, GFP_NOFS); put_page(page); @@ -675,19 +903,12 @@ static inline struct page *readpage_iter_next(struct readpages_iter *iter) for (; \ ((_page) = __readpage_next_page(&(_iter)));) \ -static void bch2_mark_pages_unalloc(struct bio *bio) -{ - struct bvec_iter iter; - struct bio_vec bv; - - bio_for_each_segment(bv, bio, iter) - page_state(bv.bv_page)->allocated = 0; -} - static void bch2_add_page_sectors(struct bio *bio, struct bkey_s_c k) { struct bvec_iter iter; struct bio_vec bv; + bool compressed = bch2_extent_is_compressed(k); + unsigned nr_ptrs = bch2_extent_nr_dirty_ptrs(k); bio_for_each_segment(bv, bio, iter) { struct bch_page_state *s = page_state(bv.bv_page); @@ -697,14 +918,14 @@ static void bch2_add_page_sectors(struct bio *bio, struct bkey_s_c k) unsigned page_sectors = min(bv.bv_len >> 9, k_sectors); - if (!s->sectors) - s->nr_replicas = bch2_extent_nr_dirty_ptrs(k); - else - s->nr_replicas = min_t(unsigned, s->nr_replicas, - bch2_extent_nr_dirty_ptrs(k)); + s->nr_replicas = !s->sectors + ? nr_ptrs + : min_t(unsigned, s->nr_replicas, nr_ptrs); BUG_ON(s->sectors + page_sectors > PAGE_SECTORS); s->sectors += page_sectors; + + s->compressed |= compressed; } } @@ -810,9 +1031,6 @@ static void bchfs_read(struct bch_fs *c, struct btree_iter *iter, if (bkey_extent_is_allocation(k.k)) bch2_add_page_sectors(bio, k); - if (!bch2_extent_is_fully_allocated(k)) - bch2_mark_pages_unalloc(bio); - if (pick.ca) { if (!is_last) { bio_inc_remaining(&rbio->bio); @@ -884,22 +1102,7 @@ static void __bchfs_readpage(struct bch_fs *c, struct bch_read_bio *rbio, { struct btree_iter iter; - /* - * Initialize page state: - * If a page is partly allocated and partly a hole, we want it to be - * marked BCH_PAGE_UNALLOCATED - so we initially mark all pages - * allocated and then mark them unallocated as we find holes: - * - * Note that the bio hasn't been split yet - it's the only bio that - * points to these pages. As we walk extents and split @bio, that - * necessarily be true, the splits won't necessarily be on page - * boundaries: - */ - struct bch_page_state *s = page_state(page); - - EBUG_ON(s->reserved); - s->allocated = 1; - s->sectors = 0; + page_state_init_for_read(page); bio_set_op_attrs(&rbio->bio, REQ_OP_READ, REQ_SYNC); bio_add_page_contig(&rbio->bio, page); @@ -922,6 +1125,40 @@ int bch2_readpage(struct file *file, struct page *page) return 0; } +static void bch2_read_single_page_end_io(struct bio *bio) +{ + complete(bio->bi_private); +} + +static int bch2_read_single_page(struct page *page, + struct address_space *mapping) +{ + struct bch_inode_info *inode = to_bch_ei(mapping->host); + struct bch_fs *c = inode->v.i_sb->s_fs_info; + struct bch_read_bio *rbio; + int ret; + DECLARE_COMPLETION_ONSTACK(done); + + rbio = rbio_init(bio_alloc_bioset(GFP_NOFS, 1, &c->bio_read), + io_opts(c, inode)); + rbio->bio.bi_private = &done; + rbio->bio.bi_end_io = bch2_read_single_page_end_io; + + __bchfs_readpage(c, rbio, inode->v.i_ino, page); + wait_for_completion(&done); + + ret = blk_status_to_errno(rbio->bio.bi_status); + bio_put(&rbio->bio); + + if (ret < 0) + return ret; + + SetPageUptodate(page); + return 0; +} + +/* writepages: */ + struct bch_writepage_state { struct bch_writepage_io *io; struct bch_io_opts opts; @@ -953,43 +1190,32 @@ static void bch2_writepage_io_done(struct closure *cl) atomic_sub(bio->bi_vcnt, &c->writeback_pages); wake_up(&c->writeback_wait); - bio_for_each_segment_all(bvec, bio, i) { - struct page *page = bvec->bv_page; - - if (io->op.op.error) { - SetPageError(page); - if (page->mapping) - set_bit(AS_EIO, &page->mapping->flags); - } - - if (io->op.op.written >= PAGE_SECTORS) { - struct bch_page_state old, new; - - old = page_state_cmpxchg(page_state(page), new, { - new.sectors = PAGE_SECTORS; - new.dirty_sectors = 0; - }); - - io->op.sectors_added -= old.dirty_sectors; - io->op.op.written -= PAGE_SECTORS; - } + if (io->op.op.error) { + bio_for_each_segment_all(bvec, bio, i) + SetPageError(bvec->bv_page); + set_bit(AS_EIO, &io->op.inode->v.i_mapping->flags); } /* * racing with fallocate can cause us to add fewer sectors than * expected - but we shouldn't add more sectors than expected: - * + */ + BUG_ON(io->op.sectors_added > (s64) io->new_sectors); + + /* * (error (due to going RO) halfway through a page can screw that up * slightly) + * XXX wtf? + BUG_ON(io->op.sectors_added - io->new_sectors >= (s64) PAGE_SECTORS); */ - BUG_ON(io->op.sectors_added >= (s64) PAGE_SECTORS); /* * PageWriteback is effectively our ref on the inode - fixup i_blocks * before calling end_page_writeback: */ - if (io->op.sectors_added) - i_sectors_acct(c, io->op.inode, io->op.sectors_added); + if (io->op.sectors_added != io->new_sectors) + i_sectors_acct(c, io->op.inode, NULL, + io->op.sectors_added - (s64) io->new_sectors); bio_for_each_segment_all(bvec, bio, i) end_page_writeback(bvec->bv_page); @@ -1017,7 +1243,7 @@ static void bch2_writepage_io_alloc(struct bch_fs *c, struct bch_writepage_state *w, struct bch_inode_info *inode, struct page *page, - struct bch_page_state s) + unsigned nr_replicas) { struct bch_write_op *op; u64 offset = (u64) page->index << PAGE_SECTOR_SHIFT; @@ -1026,13 +1252,13 @@ static void bch2_writepage_io_alloc(struct bch_fs *c, BIO_MAX_PAGES, &c->writepage_bioset), struct bch_writepage_io, op.op.wbio.bio); - op = &w->io->op.op; closure_init(&w->io->cl, NULL); - + w->io->new_sectors = 0; bch2_fswrite_op_init(&w->io->op, c, inode, w->opts, false); - op->nr_replicas = s.nr_replicas; - op->res.nr_replicas = s.nr_replicas; + op = &w->io->op.op; + op->nr_replicas = nr_replicas; + op->res.nr_replicas = nr_replicas; op->write_point = writepoint_hashed(inode->ei_last_dirtied); op->pos = POS(inode->v.i_ino, offset); op->wbio.bio.bi_iter.bi_sector = offset; @@ -1074,28 +1300,33 @@ do_io: old = page_state_cmpxchg(page_state(page), new, { EBUG_ON(!new.reserved && (new.sectors != PAGE_SECTORS || - !new.allocated)); + new.compressed)); - if (new.allocated && w->opts.compression) - new.allocated = 0; - else if (!new.reserved) - break; + if (new.reserved) + new.nr_replicas = new.reservation_replicas; new.reserved = 0; + + new.compressed |= w->opts.compression != 0; + + new.sectors += new.dirty_sectors; + new.dirty_sectors = 0; }); if (w->io && - (w->io->op.op.res.nr_replicas != old.nr_replicas || + (w->io->op.op.res.nr_replicas != new.nr_replicas || !bio_can_add_page_contig(&w->io->op.op.wbio.bio, page))) bch2_writepage_do_io(w); if (!w->io) - bch2_writepage_io_alloc(c, w, inode, page, old); + bch2_writepage_io_alloc(c, w, inode, page, new.nr_replicas); + + w->io->new_sectors += new.sectors - old.sectors; BUG_ON(inode != w->io->op.inode); BUG_ON(bio_add_page_contig(&w->io->op.op.wbio.bio, page)); if (old.reserved) - w->io->op.op.res.sectors += old.nr_replicas * PAGE_SECTORS; + w->io->op.op.res.sectors += old.reservation_replicas * PAGE_SECTORS; /* while page is locked: */ w->io->op.new_i_size = i_size; @@ -1273,37 +1504,7 @@ int bch2_writepage(struct page *page, struct writeback_control *wbc) return ret; } -static void bch2_read_single_page_end_io(struct bio *bio) -{ - complete(bio->bi_private); -} - -static int bch2_read_single_page(struct page *page, - struct address_space *mapping) -{ - struct bch_inode_info *inode = to_bch_ei(mapping->host); - struct bch_fs *c = inode->v.i_sb->s_fs_info; - struct bch_read_bio *rbio; - int ret; - DECLARE_COMPLETION_ONSTACK(done); - - rbio = rbio_init(bio_alloc_bioset(GFP_NOFS, 1, &c->bio_read), - io_opts(c, inode)); - rbio->bio.bi_private = &done; - rbio->bio.bi_end_io = bch2_read_single_page_end_io; - - __bchfs_readpage(c, rbio, inode->v.i_ino, page); - wait_for_completion(&done); - - ret = blk_status_to_errno(rbio->bio.bi_status); - bio_put(&rbio->bio); - - if (ret < 0) - return ret; - - SetPageUptodate(page); - return 0; -} +/* buffered writes: */ int bch2_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, @@ -1348,7 +1549,7 @@ readpage: if (ret) goto err; out: - ret = bch2_get_page_reservation(c, page, true); + ret = bch2_get_page_reservation(c, inode, page, true); if (ret) { if (!PageUptodate(page)) { /* @@ -1374,11 +1575,11 @@ err_unlock: return ret; } -int bch2_write_end(struct file *filp, struct address_space *mapping, +int bch2_write_end(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata) { - struct bch_inode_info *inode = to_bch_ei(page->mapping->host); + struct bch_inode_info *inode = to_bch_ei(mapping->host); struct bch_fs *c = inode->v.i_sb->s_fs_info; lockdep_assert_held(&inode->v.i_rwsem); @@ -1405,7 +1606,7 @@ int bch2_write_end(struct file *filp, struct address_space *mapping, inode->ei_last_dirtied = (unsigned long) current; } else { - bch2_put_page_reservation(c, page); + bch2_put_page_reservation(c, inode, page); } unlock_page(page); @@ -1415,7 +1616,7 @@ int bch2_write_end(struct file *filp, struct address_space *mapping, return copied; } -/* O_DIRECT */ +/* O_DIRECT reads */ static void bch2_dio_read_complete(struct closure *cl) { @@ -1529,14 +1730,15 @@ start: } } +/* O_DIRECT writes */ + static void bch2_dio_write_loop_async(struct closure *); static long bch2_dio_write_loop(struct dio_write *dio) { struct kiocb *req = dio->req; - struct file *file = req->ki_filp; - struct address_space *mapping = file->f_mapping; - struct bch_inode_info *inode = file_bch_inode(file); + struct address_space *mapping = req->ki_filp->f_mapping; + struct bch_inode_info *inode = dio->iop.inode; struct bio *bio = &dio->iop.op.wbio.bio; struct bio_vec *bv; bool sync; @@ -1615,8 +1817,8 @@ loop: ret = dio->iop.op.error ?: ((long) dio->iop.op.written << 9); err: __pagecache_block_put(&mapping->add_lock); - inode_dio_end(&inode->v); bch2_disk_reservation_put(dio->iop.op.c, &dio->iop.op.res); + bch2_quota_reservation_put(dio->iop.op.c, inode, &dio->quota_res); if (dio->free_iov) kfree(dio->iter.iov); @@ -1626,6 +1828,9 @@ err: sync = dio->sync; bio_put(bio); + /* inode->i_dio_count is our ref on inode and thus bch_fs */ + inode_dio_end(&inode->v); + if (!sync) { req->ki_complete(req, ret, 0); ret = -EIOCBQUEUED; @@ -1671,6 +1876,7 @@ static int bch2_direct_IO_write(struct kiocb *req, dio->sync = is_sync_kiocb(req) || offset + iter->count > inode->v.i_size; dio->free_iov = false; + dio->quota_res.sectors = 0; dio->iter = *iter; bch2_fswrite_op_init(&dio->iop, c, inode, io_opts(c, inode), true); dio->iop.op.write_point = writepoint_hashed((unsigned long) dio->task); @@ -1680,7 +1886,13 @@ static int bch2_direct_IO_write(struct kiocb *req, !c->opts.journal_flush_disabled) dio->iop.op.flags |= BCH_WRITE_FLUSH; - ret = bch2_disk_reservation_get(c, &dio->iop.op.res, iter->count >> 9, 0); + ret = bch2_quota_reservation_add(c, inode, &dio->quota_res, + iter->count >> 9, true); + if (unlikely(ret)) + goto err; + + ret = bch2_disk_reservation_get(c, &dio->iop.op.res, iter->count >> 9, + c->opts.data_replicas, 0); if (unlikely(ret)) { if (bch2_check_range_allocated(c, POS(inode->v.i_ino, offset >> 9), @@ -1695,6 +1907,7 @@ static int bch2_direct_IO_write(struct kiocb *req, return bch2_dio_write_loop(dio); err: bch2_disk_reservation_put(c, &dio->iop.op.res); + bch2_quota_reservation_put(c, inode, &dio->quota_res); closure_debug_destroy(&dio->cl); bio_put(bio); return ret; @@ -1765,94 +1978,7 @@ ssize_t bch2_write_iter(struct kiocb *iocb, struct iov_iter *from) return ret; } -int bch2_page_mkwrite(struct vm_fault *vmf) -{ - struct page *page = vmf->page; - struct file *file = vmf->vma->vm_file; - struct bch_inode_info *inode = file_bch_inode(file); - struct address_space *mapping = inode->v.i_mapping; - struct bch_fs *c = inode->v.i_sb->s_fs_info; - int ret = VM_FAULT_LOCKED; - - sb_start_pagefault(inode->v.i_sb); - file_update_time(file); - - /* - * Not strictly necessary, but helps avoid dio writes livelocking in - * write_invalidate_inode_pages_range() - can drop this if/when we get - * a write_invalidate_inode_pages_range() that works without dropping - * page lock before invalidating page - */ - if (current->pagecache_lock != &mapping->add_lock) - pagecache_add_get(&mapping->add_lock); - - lock_page(page); - if (page->mapping != mapping || - page_offset(page) > i_size_read(&inode->v)) { - unlock_page(page); - ret = VM_FAULT_NOPAGE; - goto out; - } - - if (bch2_get_page_reservation(c, page, true)) { - unlock_page(page); - ret = VM_FAULT_SIGBUS; - goto out; - } - - if (!PageDirty(page)) - set_page_dirty(page); - wait_for_stable_page(page); -out: - if (current->pagecache_lock != &mapping->add_lock) - pagecache_add_put(&mapping->add_lock); - sb_end_pagefault(inode->v.i_sb); - return ret; -} - -void bch2_invalidatepage(struct page *page, unsigned int offset, - unsigned int length) -{ - EBUG_ON(!PageLocked(page)); - EBUG_ON(PageWriteback(page)); - - if (offset || length < PAGE_SIZE) - return; - - bch2_clear_page_bits(page); -} - -int bch2_releasepage(struct page *page, gfp_t gfp_mask) -{ - EBUG_ON(!PageLocked(page)); - EBUG_ON(PageWriteback(page)); - - if (PageDirty(page)) - return 0; - - bch2_clear_page_bits(page); - return 1; -} - -#ifdef CONFIG_MIGRATION -int bch2_migrate_page(struct address_space *mapping, struct page *newpage, - struct page *page, enum migrate_mode mode) -{ - int ret; - - ret = migrate_page_move_mapping(mapping, newpage, page, NULL, mode, 0); - if (ret != MIGRATEPAGE_SUCCESS) - return ret; - - if (PagePrivate(page)) { - *page_state(newpage) = *page_state(page); - ClearPagePrivate(page); - } - - migrate_page_copy(newpage, page); - return MIGRATEPAGE_SUCCESS; -} -#endif +/* fsync: */ int bch2_fsync(struct file *file, loff_t start, loff_t end, int datasync) { @@ -1870,6 +1996,8 @@ int bch2_fsync(struct file *file, loff_t start, loff_t end, int datasync) return bch2_journal_flush_seq(&c->journal, inode->ei_journal_seq); } +/* truncate: */ + static int __bch2_truncate_page(struct bch_inode_info *inode, pgoff_t index, loff_t start, loff_t end) { @@ -1934,7 +2062,7 @@ create: * XXX: because we aren't currently tracking whether the page has actual * data in it (vs. just 0s, or only partially written) this wrong. ick. */ - ret = bch2_get_page_reservation(c, page, false); + ret = bch2_get_page_reservation(c, inode, page, false); BUG_ON(ret); if (index == start >> PAGE_SHIFT && @@ -2023,6 +2151,8 @@ err_put_pagecache: return ret; } +/* fallocate: */ + static long bch2_fpunch(struct bch_inode_info *inode, loff_t offset, loff_t len) { struct bch_fs *c = inode->v.i_sb->s_fs_info; @@ -2054,7 +2184,14 @@ static long bch2_fpunch(struct bch_inode_info *inode, loff_t offset, loff_t len) truncate_pagecache_range(&inode->v, offset, offset + len - 1); if (discard_start < discard_end) { - struct disk_reservation disk_res; + /* + * We need to pass in a disk reservation here because we might + * be splitting a compressed extent into two. This isn't a + * problem with truncate because truncate will never split an + * extent, only truncate it... + */ + struct disk_reservation disk_res = + bch2_disk_reservation_init(c, 0); struct i_sectors_hook i_sectors_hook = i_sectors_hook_init(inode, 0); int ret; @@ -2063,15 +2200,6 @@ static long bch2_fpunch(struct bch_inode_info *inode, loff_t offset, loff_t len) if (unlikely(ret)) goto err; - /* - * We need to pass in a disk reservation here because we might - * be splitting a compressed extent into two. This isn't a - * problem with truncate because truncate will never split an - * extent, only truncate it... - */ - ret = bch2_disk_reservation_get(c, &disk_res, 0, 0); - BUG_ON(ret); - ret = bch2_btree_delete_range(c, BTREE_ID_EXTENTS, POS(ino, discard_start), @@ -2080,7 +2208,6 @@ static long bch2_fpunch(struct bch_inode_info *inode, loff_t offset, loff_t len) &disk_res, &i_sectors_hook.hook, &inode->ei_journal_seq); - bch2_disk_reservation_put(c, &disk_res); ret = i_sectors_dirty_finish(c, &i_sectors_hook) ?: ret; } @@ -2168,7 +2295,8 @@ static long bch2_fcollapse(struct bch_inode_info *inode, BUG_ON(bkey_cmp(dst.pos, bkey_start_pos(©.k.k))); ret = bch2_disk_reservation_get(c, &disk_res, copy.k.k.size, - BCH_DISK_RESERVATION_NOFAIL); + bch2_extent_nr_dirty_ptrs(bkey_i_to_s_c(©.k)), + BCH_DISK_RESERVATION_NOFAIL); BUG_ON(ret); ret = bch2_btree_insert_at(c, &disk_res, &i_sectors_hook.hook, @@ -2304,11 +2432,19 @@ static long bch2_fallocate(struct bch_inode_info *inode, int mode, sectors = reservation.k.size; reservation.v.nr_replicas = bch2_extent_nr_dirty_ptrs(k); + if (!bkey_extent_is_allocation(k.k)) { + ret = bch2_quota_reservation_add(c, inode, + &i_sectors_hook.quota_res, + sectors, true); + if (unlikely(ret)) + goto err_put_sectors_dirty; + } + if (reservation.v.nr_replicas < replicas || bch2_extent_is_compressed(k)) { - ret = bch2_disk_reservation_get(c, &disk_res, - sectors, 0); - if (ret) + ret = bch2_disk_reservation_get(c, &disk_res, sectors, + replicas, 0); + if (unlikely(ret)) goto err_put_sectors_dirty; reservation.v.nr_replicas = disk_res.nr_replicas; @@ -2385,6 +2521,8 @@ long bch2_fallocate_dispatch(struct file *file, int mode, return -EOPNOTSUPP; } +/* fseek: */ + static bool page_is_data(struct page *page) { /* XXX: should only have to check PageDirty */ diff --git a/libbcachefs/fs-ioctl.c b/libbcachefs/fs-ioctl.c index 6ae67f92..bfbe574d 100644 --- a/libbcachefs/fs-ioctl.c +++ b/libbcachefs/fs-ioctl.c @@ -173,7 +173,8 @@ static int bch2_set_projid(struct bch_fs *c, qid.q[QTYP_PRJ] = projid; ret = bch2_quota_transfer(c, 1 << QTYP_PRJ, qid, inode->ei_qid, - inode->v.i_blocks); + inode->v.i_blocks + + inode->ei_quota_reserved); if (ret) return ret; diff --git a/libbcachefs/fs.c b/libbcachefs/fs.c index 8869ba0f..aba845b2 100644 --- a/libbcachefs/fs.c +++ b/libbcachefs/fs.c @@ -217,10 +217,8 @@ static struct bch_inode_info *bch2_vfs_inode_create(struct bch_fs *c, #ifdef CONFIG_BCACHEFS_POSIX_ACL ret = posix_acl_create(&dir->v, &inode->v.i_mode, &default_acl, &acl); - if (ret) { - make_bad_inode(&inode->v); + if (ret) goto err_make_bad; - } #endif bch2_inode_init(c, &inode_u, @@ -232,20 +230,17 @@ static struct bch_inode_info *bch2_vfs_inode_create(struct bch_fs *c, inode_u.bi_project = dir->ei_qid.q[QTYP_PRJ]; ret = bch2_quota_acct(c, bch_qid(&inode_u), Q_INO, 1, BCH_QUOTA_PREALLOC); - if (ret) { - make_bad_inode(&inode->v); + if (ret) goto err_make_bad; - } ret = bch2_inode_create(c, &inode_u, BLOCKDEV_INODE_MAX, 0, &c->unused_inode_hint); - if (unlikely(ret)) { - bch2_quota_acct(c, bch_qid(&inode_u), Q_INO, -1, BCH_QUOTA_WARN); - goto err_make_bad; - } + if (unlikely(ret)) + goto err_acct_quota; bch2_vfs_inode_init(c, inode, &inode_u); + atomic_long_inc(&c->nr_inodes); if (default_acl) { ret = bch2_set_acl(&inode->v, default_acl, ACL_TYPE_DEFAULT); @@ -260,11 +255,12 @@ static struct bch_inode_info *bch2_vfs_inode_create(struct bch_fs *c, } insert_inode_hash(&inode->v); - atomic_long_inc(&c->nr_inodes); out: posix_acl_release(default_acl); posix_acl_release(acl); return inode; +err_acct_quota: + bch2_quota_acct(c, bch_qid(&inode_u), Q_INO, -1, BCH_QUOTA_WARN); err_make_bad: /* * indicate to bch_evict_inode that the inode was never actually @@ -643,7 +639,8 @@ static int bch2_setattr_nonsize(struct bch_inode_info *inode, struct iattr *iatt if (qtypes) { ret = bch2_quota_transfer(c, qtypes, qid, inode->ei_qid, - inode->v.i_blocks); + inode->v.i_blocks + + inode->ei_quota_reserved); if (ret) goto out_unlock; } @@ -953,6 +950,7 @@ static void bch2_vfs_inode_init(struct bch_fs *c, inode->v.i_ctime = bch2_time_to_timespec(c, bi->bi_ctime); inode->ei_journal_seq = 0; + inode->ei_quota_reserved = 0; inode->ei_qid = bch_qid(bi); inode->ei_str_hash = bch2_hash_info_init(c, bi); inode->ei_inode = *bi; @@ -1038,6 +1036,8 @@ static void bch2_evict_inode(struct inode *vinode) clear_inode(&inode->v); + BUG_ON(!is_bad_inode(&inode->v) && inode->ei_quota_reserved); + if (!inode->v.i_nlink && !is_bad_inode(&inode->v)) { bch2_quota_acct(c, inode->ei_qid, Q_SPC, -((s64) inode->v.i_blocks), BCH_QUOTA_WARN); diff --git a/libbcachefs/fs.h b/libbcachefs/fs.h index dd0bd4ef..fddfb2d2 100644 --- a/libbcachefs/fs.h +++ b/libbcachefs/fs.h @@ -13,6 +13,7 @@ struct bch_inode_info { struct mutex ei_update_lock; u64 ei_journal_seq; + u64 ei_quota_reserved; unsigned long ei_last_dirtied; struct bch_qid ei_qid; diff --git a/libbcachefs/journal.c b/libbcachefs/journal.c index 811f7a5c..1952643f 100644 --- a/libbcachefs/journal.c +++ b/libbcachefs/journal.c @@ -1469,6 +1469,8 @@ void bch2_journal_start(struct bch_fs *c) journal_pin_new_entry(j, 1); bch2_journal_buf_init(j); + spin_unlock(&j->lock); + /* * Adding entries to the next journal entry before allocating space on * disk for the next journal entry - this is ok, because these entries @@ -1487,8 +1489,6 @@ void bch2_journal_start(struct bch_fs *c) bl->written = true; } - spin_unlock(&j->lock); - queue_delayed_work(system_freezable_wq, &j->reclaim_work, 0); } @@ -1505,7 +1505,6 @@ int bch2_journal_replay(struct bch_fs *c, struct list_head *list) journal_seq_pin(j, le64_to_cpu(i->j.seq)); for_each_jset_key(k, _n, entry, &i->j) { - struct disk_reservation disk_res; if (entry->btree_id == BTREE_ID_ALLOC) { /* @@ -1514,19 +1513,18 @@ int bch2_journal_replay(struct bch_fs *c, struct list_head *list) */ ret = bch2_alloc_replay_key(c, k->k.p); } else { - /* * We might cause compressed extents to be * split, so we need to pass in a * disk_reservation: */ - BUG_ON(bch2_disk_reservation_get(c, &disk_res, 0, 0)); + struct disk_reservation disk_res = + bch2_disk_reservation_init(c, 0); ret = bch2_btree_insert(c, entry->btree_id, k, &disk_res, NULL, NULL, BTREE_INSERT_NOFAIL| BTREE_INSERT_JOURNAL_REPLAY); - bch2_disk_reservation_put(c, &disk_res); } if (ret) { @@ -1580,7 +1578,7 @@ static int bch2_set_nr_journal_buckets(struct bch_fs *c, struct bch_dev *ca, */ if (bch2_disk_reservation_get(c, &disk_res, - bucket_to_sector(ca, nr - ja->nr), 0)) + bucket_to_sector(ca, nr - ja->nr), 1, 0)) return -ENOSPC; mutex_lock(&c->sb_lock); diff --git a/libbcachefs/quota.c b/libbcachefs/quota.c index c550fd9e..854f7f55 100644 --- a/libbcachefs/quota.c +++ b/libbcachefs/quota.c @@ -749,14 +749,14 @@ static int bch2_set_quota(struct super_block *sb, struct kqid qid, } if (qdq->d_fieldmask & QC_SPC_SOFT) - new_quota.v.c[Q_SPC].softlimit = cpu_to_le64(qdq->d_spc_softlimit); + new_quota.v.c[Q_SPC].softlimit = cpu_to_le64(qdq->d_spc_softlimit >> 9); if (qdq->d_fieldmask & QC_SPC_HARD) - new_quota.v.c[Q_SPC].hardlimit = cpu_to_le64(qdq->d_spc_hardlimit); + new_quota.v.c[Q_SPC].hardlimit = cpu_to_le64(qdq->d_spc_hardlimit >> 9); if (qdq->d_fieldmask & QC_INO_SOFT) - new_quota.v.c[Q_INO].softlimit = cpu_to_le64(qdq->d_spc_softlimit); + new_quota.v.c[Q_INO].softlimit = cpu_to_le64(qdq->d_ino_softlimit); if (qdq->d_fieldmask & QC_INO_HARD) - new_quota.v.c[Q_INO].hardlimit = cpu_to_le64(qdq->d_spc_hardlimit); + new_quota.v.c[Q_INO].hardlimit = cpu_to_le64(qdq->d_ino_hardlimit); ret = bch2_btree_insert_at(c, NULL, NULL, NULL, 0, BTREE_INSERT_ENTRY(&iter, &new_quota.k_i)); diff --git a/libbcachefs/quota.h b/libbcachefs/quota.h index 09d51a83..b5536be9 100644 --- a/libbcachefs/quota.h +++ b/libbcachefs/quota.h @@ -36,8 +36,19 @@ extern const struct quotactl_ops bch2_quotactl_operations; #else -#define bch2_quota_acct(_c, _uid, _gid, _counter, _v) (0) -#define bch2_quota_transfer(_c, _type, _src, _dst, _v) (0) +static inline int bch2_quota_acct(struct bch_fs *c, struct bch_qid qid, + enum quota_counters counter, s64 v, + enum quota_acct_mode mode) +{ + return 0; +} + +static inline int bch2_quota_transfer(struct bch_fs *c, unsigned qtypes, + struct bch_qid dst, + struct bch_qid src, u64 space) +{ + return 0; +} static inline void bch2_fs_quota_exit(struct bch_fs *c) {} static inline void bch2_fs_quota_init(struct bch_fs *c) {}