diff --git a/.bcachefs_revision b/.bcachefs_revision index b5f30cd3..610413ad 100644 --- a/.bcachefs_revision +++ b/.bcachefs_revision @@ -1 +1 @@ -43a464c9dd38b50c1a89845366f838fe70fbb743 +c8aa081de3a61efdabaf70cbfb1e92c6c1d92e76 diff --git a/libbcachefs/btree_iter.c b/libbcachefs/btree_iter.c index 263cd18f..ac23a251 100644 --- a/libbcachefs/btree_iter.c +++ b/libbcachefs/btree_iter.c @@ -1635,16 +1635,6 @@ static void bch2_btree_iter_link(struct btree_iter *iter, struct btree_iter *new new->next = iter->next; iter->next = new; - - if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG)) { - unsigned nr_iters = 0; - - for_each_btree_iter(new, iter) - if (iter->btree_id == new->btree_id) - nr_iters++; - - BUG_ON(nr_iters > SIX_LOCK_MAX_RECURSE); - } } void bch2_btree_iter_copy(struct btree_iter *dst, struct btree_iter *src) @@ -1681,7 +1671,7 @@ static inline unsigned btree_trans_iter_idx(struct btree_trans *trans, ssize_t idx = iter - trans->iters; BUG_ON(idx < 0 || idx >= trans->nr_iters); - BUG_ON(!(trans->iters_live & (1U << idx))); + BUG_ON(!(trans->iters_live & (1ULL << idx))); return idx; } @@ -1691,7 +1681,7 @@ void bch2_trans_iter_put(struct btree_trans *trans, { ssize_t idx = btree_trans_iter_idx(trans, iter); - trans->iters_live &= ~(1U << idx); + trans->iters_live &= ~(1ULL << idx); } void bch2_trans_iter_free(struct btree_trans *trans, @@ -1699,23 +1689,50 @@ void bch2_trans_iter_free(struct btree_trans *trans, { ssize_t idx = btree_trans_iter_idx(trans, iter); - trans->iters_live &= ~(1U << idx); - trans->iters_linked &= ~(1U << idx); + trans->iters_live &= ~(1ULL << idx); + trans->iters_linked &= ~(1ULL << idx); bch2_btree_iter_unlink(iter); } -static int btree_trans_realloc_iters(struct btree_trans *trans) +static int btree_trans_realloc_iters(struct btree_trans *trans, + unsigned new_size) { - struct btree_iter *new_iters; + void *new_iters, *new_updates; unsigned i; + BUG_ON(new_size > BTREE_ITER_MAX); + + if (new_size <= trans->size) + return 0; + + BUG_ON(trans->used_mempool); + bch2_trans_unlock(trans); + new_iters = kmalloc(sizeof(struct btree_iter) * new_size + + sizeof(struct btree_insert_entry) * (new_size + 4), + GFP_NOFS); + if (new_iters) + goto success; + new_iters = mempool_alloc(&trans->c->btree_iters_pool, GFP_NOFS); + new_size = BTREE_ITER_MAX; + + trans->used_mempool = true; +success: + new_updates = new_iters + sizeof(struct btree_iter) * new_size; memcpy(new_iters, trans->iters, sizeof(struct btree_iter) * trans->nr_iters); - trans->iters = new_iters; + memcpy(new_updates, trans->updates, + sizeof(struct btree_insert_entry) * trans->nr_updates); + + if (trans->iters != trans->iters_onstack) + kfree(trans->iters); + + trans->iters = new_iters; + trans->updates = new_updates; + trans->size = new_size; for (i = 0; i < trans->nr_iters; i++) trans->iters[i].next = &trans->iters[i]; @@ -1741,8 +1758,7 @@ static int btree_trans_realloc_iters(struct btree_trans *trans) void bch2_trans_preload_iters(struct btree_trans *trans) { - if (trans->iters == trans->iters_onstack) - btree_trans_realloc_iters(trans); + btree_trans_realloc_iters(trans, 16); } static struct btree_iter *__btree_trans_get_iter(struct btree_trans *trans, @@ -1755,7 +1771,7 @@ static struct btree_iter *__btree_trans_get_iter(struct btree_trans *trans, BUG_ON(trans->nr_iters > BTREE_ITER_MAX); for (idx = 0; idx < trans->nr_iters; idx++) - if (trans->iter_ids[idx] == iter_id) + if (trans->iters[idx].id == iter_id) goto found; idx = -1; found: @@ -1764,19 +1780,20 @@ found: if (idx < trans->nr_iters) goto got_slot; - BUG_ON(trans->nr_iters == BTREE_ITER_MAX); + BUG_ON(trans->nr_iters > trans->size); - if (trans->iters == trans->iters_onstack && - trans->nr_iters == ARRAY_SIZE(trans->iters_onstack)) { - int ret = btree_trans_realloc_iters(trans); + if (trans->nr_iters == trans->size) { + int ret = btree_trans_realloc_iters(trans, + trans->size * 2); if (ret) return ERR_PTR(ret); } idx = trans->nr_iters++; + BUG_ON(trans->nr_iters > trans->size); got_slot: - trans->iter_ids[idx] = iter_id; iter = &trans->iters[idx]; + iter->id = iter_id; bch2_btree_iter_init(iter, trans->c, btree_id, POS_MIN, flags); } else { @@ -1786,15 +1803,15 @@ got_slot: iter->flags |= flags & (BTREE_ITER_INTENT|BTREE_ITER_PREFETCH); } - BUG_ON(trans->iters_live & (1 << idx)); - trans->iters_live |= 1 << idx; + BUG_ON(trans->iters_live & (1ULL << idx)); + trans->iters_live |= 1ULL << idx; if (trans->iters_linked && !(trans->iters_linked & (1 << idx))) bch2_btree_iter_link(&trans->iters[__ffs(trans->iters_linked)], iter); - trans->iters_linked |= 1 << idx; + trans->iters_linked |= 1ULL << idx; btree_trans_verify(trans); @@ -1878,6 +1895,7 @@ int bch2_trans_unlock(struct btree_trans *trans) void __bch2_trans_begin(struct btree_trans *trans) { + u64 linked_not_live; unsigned idx; btree_trans_verify(trans); @@ -1890,10 +1908,15 @@ void __bch2_trans_begin(struct btree_trans *trans) * further (allocated an iter with a higher idx) than where the iter * was originally allocated: */ - while (trans->iters_linked && - trans->iters_live && - (idx = __fls(trans->iters_linked)) > - __fls(trans->iters_live)) { + while (1) { + linked_not_live = trans->iters_linked & ~trans->iters_live; + if (!linked_not_live) + break; + + idx = __ffs64(linked_not_live); + if (1ULL << idx > trans->iters_live) + break; + trans->iters_linked ^= 1 << idx; bch2_btree_iter_unlink(&trans->iters[idx]); } @@ -1907,16 +1930,12 @@ void __bch2_trans_begin(struct btree_trans *trans) void bch2_trans_init(struct btree_trans *trans, struct bch_fs *c) { + memset(trans, 0, offsetof(struct btree_trans, iters_onstack)); + trans->c = c; - trans->nr_restarts = 0; - trans->nr_iters = 0; - trans->iters_live = 0; - trans->iters_linked = 0; - trans->nr_updates = 0; - trans->mem_top = 0; - trans->mem_bytes = 0; - trans->mem = NULL; + trans->size = ARRAY_SIZE(trans->iters_onstack); trans->iters = trans->iters_onstack; + trans->updates = trans->updates_onstack; } int bch2_trans_exit(struct btree_trans *trans) @@ -1924,8 +1943,10 @@ int bch2_trans_exit(struct btree_trans *trans) int ret = bch2_trans_unlock(trans); kfree(trans->mem); - if (trans->iters != trans->iters_onstack) + if (trans->used_mempool) mempool_free(trans->iters, &trans->c->btree_iters_pool); + else if (trans->iters != trans->iters_onstack) + kfree(trans->iters); trans->mem = (void *) 0x1; trans->iters = (void *) 0x1; return ret; diff --git a/libbcachefs/btree_types.h b/libbcachefs/btree_types.h index 15a4d382..a6aea023 100644 --- a/libbcachefs/btree_types.h +++ b/libbcachefs/btree_types.h @@ -231,19 +231,18 @@ struct btree_iter { */ struct bkey k; + u64 id; + /* * Circular linked list of linked iterators: linked iterators share * locks (e.g. two linked iterators may have the same node intent * locked, or read and write locked, at the same time), and insertions * through one iterator won't invalidate the other linked iterators. */ - /* Must come last: */ struct btree_iter *next; }; -#define BTREE_ITER_MAX 8 - struct deferred_update { struct journal_preres res; struct journal_entry_pin journal; @@ -269,25 +268,29 @@ struct btree_insert_entry { bool deferred; }; +#define BTREE_ITER_MAX 64 + struct btree_trans { struct bch_fs *c; size_t nr_restarts; + u64 iters_live; + u64 iters_linked; + u8 nr_iters; - u8 iters_live; - u8 iters_linked; u8 nr_updates; + u8 size; + unsigned used_mempool:1; unsigned mem_top; unsigned mem_bytes; void *mem; struct btree_iter *iters; - u64 iter_ids[BTREE_ITER_MAX]; - - struct btree_insert_entry updates[BTREE_ITER_MAX]; + struct btree_insert_entry *updates; struct btree_iter iters_onstack[2]; + struct btree_insert_entry updates_onstack[6]; }; #define BTREE_FLAG(flag) \ diff --git a/libbcachefs/btree_update.h b/libbcachefs/btree_update.h index 7545f4f1..1f371b5a 100644 --- a/libbcachefs/btree_update.h +++ b/libbcachefs/btree_update.h @@ -147,7 +147,7 @@ static inline void bch2_trans_update(struct btree_trans *trans, struct btree_insert_entry entry) { - BUG_ON(trans->nr_updates >= ARRAY_SIZE(trans->updates)); + BUG_ON(trans->nr_updates >= trans->nr_iters + 4); trans->updates[trans->nr_updates++] = entry; } diff --git a/libbcachefs/fs-io.c b/libbcachefs/fs-io.c index ce46d0ef..cabd3e08 100644 --- a/libbcachefs/fs-io.c +++ b/libbcachefs/fs-io.c @@ -232,12 +232,14 @@ static void i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode, static int sum_sector_overwrites(struct btree_trans *trans, struct btree_iter *extent_iter, struct bkey_i *new, bool *allocating, - s64 *i_sectors_delta) + s64 *delta) { - struct btree_iter *iter = bch2_trans_copy_iter(trans, extent_iter); + struct btree_iter *iter; struct bkey_s_c old; - s64 delta = 0; + *delta = 0; + + iter = bch2_trans_copy_iter(trans, extent_iter); if (IS_ERR(iter)) return PTR_ERR(iter); @@ -257,7 +259,7 @@ static int sum_sector_overwrites(struct btree_trans *trans, bch2_bkey_nr_dirty_ptrs(bkey_i_to_s_c(new))) *allocating = true; - delta += (min(new->k.p.offset, + *delta += (min(new->k.p.offset, old.k->p.offset) - max(bkey_start_offset(&new->k), bkey_start_offset(old.k))) * @@ -271,8 +273,6 @@ static int sum_sector_overwrites(struct btree_trans *trans, } bch2_trans_iter_free(trans, iter); - - *i_sectors_delta = delta; return 0; } diff --git a/libbcachefs/journal.c b/libbcachefs/journal.c index e2386bb7..a58a1fb6 100644 --- a/libbcachefs/journal.c +++ b/libbcachefs/journal.c @@ -248,13 +248,13 @@ static int journal_entry_open(struct journal *j) do { old.v = new.v = v; - EBUG_ON(journal_state_count(new, new.idx)); - if (old.cur_entry_offset == JOURNAL_ENTRY_ERROR_VAL) return -EROFS; /* Handle any already added entries */ new.cur_entry_offset = le32_to_cpu(buf->data->u64s); + + EBUG_ON(journal_state_count(new, new.idx)); journal_state_inc(&new); } while ((v = atomic64_cmpxchg(&j->reservations.counter, old.v, new.v)) != old.v); diff --git a/libbcachefs/journal_reclaim.c b/libbcachefs/journal_reclaim.c index eb989462..5cee6872 100644 --- a/libbcachefs/journal_reclaim.c +++ b/libbcachefs/journal_reclaim.c @@ -501,8 +501,8 @@ void bch2_journal_reclaim(struct journal *j) nr_buckets = min(nr_buckets, ja->nr); bucket_to_flush = (ja->cur_idx + nr_buckets) % ja->nr; - seq_to_flush = max_t(u64, seq_to_flush, - ja->bucket_seq[bucket_to_flush]); + seq_to_flush = max(seq_to_flush, + ja->bucket_seq[bucket_to_flush]); } /* Also flush if the pin fifo is more than half full */ @@ -519,12 +519,14 @@ void bch2_journal_reclaim(struct journal *j) msecs_to_jiffies(j->reclaim_delay_ms))) min_nr = 1; - if (j->prereserved.reserved * 2 > j->prereserved.remaining) + if (j->prereserved.reserved * 2 > j->prereserved.remaining) { + seq_to_flush = max(seq_to_flush, journal_last_seq(j)); min_nr = 1; + } journal_flush_pins(j, seq_to_flush, min_nr); - if (!test_bit(BCH_FS_RO, &c->flags)) + if (!bch2_journal_error(j)) queue_delayed_work(c->journal_reclaim_wq, &j->reclaim_work, msecs_to_jiffies(j->reclaim_delay_ms)); } diff --git a/libbcachefs/super-io.c b/libbcachefs/super-io.c index 71d97c57..ca361424 100644 --- a/libbcachefs/super-io.c +++ b/libbcachefs/super-io.c @@ -1019,7 +1019,10 @@ void bch2_fs_mark_clean(struct bch_fs *c, bool clean) sb_clean->flags = 0; sb_clean->read_clock = cpu_to_le16(c->bucket_clock[READ].hand); sb_clean->write_clock = cpu_to_le16(c->bucket_clock[WRITE].hand); - sb_clean->journal_seq = journal_cur_seq(&c->journal) - 1; + sb_clean->journal_seq = cpu_to_le64(journal_cur_seq(&c->journal) - 1); + + /* Trying to catch outstanding bug: */ + BUG_ON(le64_to_cpu(sb_clean->journal_seq) > S64_MAX); entry = sb_clean->start; entry = bch2_journal_super_entries_add_common(c, entry); diff --git a/libbcachefs/super.c b/libbcachefs/super.c index 1528f77e..4ec8a0b5 100644 --- a/libbcachefs/super.c +++ b/libbcachefs/super.c @@ -681,7 +681,9 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) mempool_init_kvpmalloc_pool(&c->btree_bounce_pool, 1, btree_bytes(c)) || mempool_init_kmalloc_pool(&c->btree_iters_pool, 1, - sizeof(struct btree_iter) * BTREE_ITER_MAX) || + sizeof(struct btree_iter) * BTREE_ITER_MAX + + sizeof(struct btree_insert_entry) * + (BTREE_ITER_MAX + 4)) || bch2_io_clock_init(&c->io_clock[READ]) || bch2_io_clock_init(&c->io_clock[WRITE]) || bch2_fs_journal_init(&c->journal) ||