alex/bcachefs-tools
1
0
Fork 0
mirror of https://github.com/koverstreet/bcachefs-tools.git synced 2025-02-22 00:00:03 +03:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Update bcachefs sources to 6406e05835 bcachefs: Nocow support

Browse Source
This commit is contained in:
Kent Overstreet 2022-11-18 18:21:11 -05:00
parent e4716b10ed
commit c1f55a60c4
41 changed files with 1351 additions and 470 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.bcachefs_revision
View File

@ -1 +1 @@
8d3fc97ca3f24d8f7ab1e9ed04d8ca354c44dd8c
6406e0583565af5f7c51f9c43a34e5f7a92dadcd

5
libbcachefs/alloc_foreground.c
View File

@ -216,6 +216,11 @@ static struct open_bucket *__try_alloc_bucket(struct bch_fs *c, struct bch_dev *
return NULL;
}
if (bch2_bucket_nocow_is_locked(&c->nocow_locks, POS(ca->dev_idx, bucket))) {
s->skipped_nocow++;
return NULL;
}
spin_lock(&c->freelist_lock);
if (unlikely(c->open_buckets_nr_free <= open_buckets_reserved(reserve))) {

1
libbcachefs/alloc_types.h
View File

@ -13,6 +13,7 @@ struct bucket_alloc_state {
u64 buckets_seen;
u64 skipped_open;
u64 skipped_need_journal_commit;
u64 skipped_nocow;
u64 skipped_nouse;
};

64
libbcachefs/bcachefs.h
View File

@ -206,6 +206,7 @@
#include "bcachefs_format.h"
#include "errcode.h"
#include "fifo.h"
#include "nocow_locking.h"
#include "opts.h"
#include "util.h"
@ -230,15 +231,27 @@ do { \
#endif
#ifdef BCACHEFS_LOG_PREFIX
#define bch2_log_msg(_c, fmt) "bcachefs (%s): " fmt, ((_c)->name)
#define bch2_fmt(_c, fmt) bch2_log_msg(_c, fmt "\n")
#define bch2_fmt_inum(_c, _inum, fmt) "bcachefs (%s inum %llu): " fmt "\n", ((_c)->name), (_inum)
#define bch2_log_msg(_c, fmt) "bcachefs (%s): " fmt, ((_c)->name)
#define bch2_fmt_dev(_ca, fmt) "bcachefs (%s): " fmt "\n", ((_ca)->name)
#define bch2_fmt_dev_offset(_ca, _offset, fmt) "bcachefs (%s sector %llu): " fmt "\n", ((_ca)->name), (_offset)
#define bch2_fmt_inum(_c, _inum, fmt) "bcachefs (%s inum %llu): " fmt "\n", ((_c)->name), (_inum)
#define bch2_fmt_inum_offset(_c, _inum, _offset, fmt) \
"bcachefs (%s inum %llu offset %llu): " fmt "\n", ((_c)->name), (_inum), (_offset)
#else
#define bch2_log_msg(_c, fmt) fmt
#define bch2_fmt(_c, fmt) fmt "\n"
#define bch2_fmt_inum(_c, _inum, fmt) "inum %llu: " fmt "\n", (_inum)
#define bch2_log_msg(_c, fmt) fmt
#define bch2_fmt_dev(_ca, fmt) "%s: " fmt "\n", ((_ca)->name)
#define bch2_fmt_dev_offset(_ca, _offset, fmt) "%s sector %llu: " fmt "\n", ((_ca)->name), (_offset)
#define bch2_fmt_inum(_c, _inum, fmt) "inum %llu: " fmt "\n", (_inum)
#define bch2_fmt_inum_offset(_c, _inum, _offset, fmt) \
"inum %llu offset %llu: " fmt "\n", (_inum), (_offset)
#endif
#define bch2_fmt(_c, fmt) bch2_log_msg(_c, fmt "\n")
#define bch_info(c, fmt, ...) \
printk(KERN_INFO bch2_fmt(c, fmt), ##__VA_ARGS__)
#define bch_notice(c, fmt, ...) \
@ -247,13 +260,28 @@ do { \
printk(KERN_WARNING bch2_fmt(c, fmt), ##__VA_ARGS__)
#define bch_warn_ratelimited(c, fmt, ...) \
printk_ratelimited(KERN_WARNING bch2_fmt(c, fmt), ##__VA_ARGS__)
#define bch_err(c, fmt, ...) \
printk(KERN_ERR bch2_fmt(c, fmt), ##__VA_ARGS__)
#define bch_err_dev(ca, fmt, ...) \
printk(KERN_ERR bch2_fmt_dev(ca, fmt), ##__VA_ARGS__)
#define bch_err_dev_offset(ca, _offset, fmt, ...) \
printk(KERN_ERR bch2_fmt_dev_offset(ca, _offset, fmt), ##__VA_ARGS__)
#define bch_err_inum(c, _inum, fmt, ...) \
printk(KERN_ERR bch2_fmt_inum(c, _inum, fmt), ##__VA_ARGS__)
#define bch_err_inum_offset(c, _inum, _offset, fmt, ...) \
printk(KERN_ERR bch2_fmt_inum_offset(c, _inum, _offset, fmt), ##__VA_ARGS__)
#define bch_err_ratelimited(c, fmt, ...) \
printk_ratelimited(KERN_ERR bch2_fmt(c, fmt), ##__VA_ARGS__)
#define bch_err_dev_ratelimited(ca, fmt, ...) \
printk_ratelimited(KERN_ERR bch2_fmt_dev(ca, fmt), ##__VA_ARGS__)
#define bch_err_dev_offset_ratelimited(ca, _offset, fmt, ...) \
printk_ratelimited(KERN_ERR bch2_fmt_dev_offset(ca, _offset, fmt), ##__VA_ARGS__)
#define bch_err_inum_ratelimited(c, _inum, fmt, ...) \
printk_ratelimited(KERN_ERR bch2_fmt_inum(c, _inum, fmt), ##__VA_ARGS__)
#define bch_err_inum_offset_ratelimited(c, _inum, _offset, fmt, ...) \
printk_ratelimited(KERN_ERR bch2_fmt_inum_offset(c, _inum, _offset, fmt), ##__VA_ARGS__)
#define bch_verbose(c, fmt, ...) \
do { \
@ -349,7 +377,8 @@ BCH_DEBUG_PARAMS_DEBUG()
x(journal_flush_seq) \
x(blocked_journal) \
x(blocked_allocate) \
x(blocked_allocate_open_bucket)
x(blocked_allocate_open_bucket) \
x(nocow_lock_contended)
enum bch_time_stats {
#define x(name) BCH_TIME_##name,
@ -448,6 +477,7 @@ struct bch_dev {
struct bch_sb *sb_read_scratch;
int sb_write_error;
dev_t dev;
atomic_t flush_seq;
struct bch_devs_mask self;
@ -602,23 +632,6 @@ typedef struct {
#define BCACHEFS_ROOT_SUBVOL_INUM \
((subvol_inum) { BCACHEFS_ROOT_SUBVOL, BCACHEFS_ROOT_INO })
#define BCH_BTREE_WRITE_TYPES() \
x(initial, 0) \
x(init_next_bset, 1) \
x(cache_reclaim, 2) \
x(journal_reclaim, 3) \
x(interior, 4)
enum btree_write_type {
#define x(t, n) BTREE_WRITE_##t,
BCH_BTREE_WRITE_TYPES()
#undef x
BTREE_WRITE_TYPE_NR,
};
#define BTREE_WRITE_TYPE_MASK (roundup_pow_of_two(BTREE_WRITE_TYPE_NR) - 1)
#define BTREE_WRITE_TYPE_BITS ilog2(BTREE_WRITE_TYPE_MASK)
struct bch_fs {
struct closure cl;
@ -847,6 +860,8 @@ struct bch_fs {
struct bio_set bio_write;
struct mutex bio_bounce_pages_lock;
mempool_t bio_bounce_pages;
struct bucket_nocow_lock_table
nocow_locks;
struct rhashtable promote_table;
mempool_t compression_bounce[2];
@ -908,6 +923,7 @@ struct bch_fs {
struct bio_set writepage_bioset;
struct bio_set dio_write_bioset;
struct bio_set dio_read_bioset;
struct bio_set nocow_flush_bioset;
/* ERRORS */
struct list_head fsck_errors;

14
libbcachefs/bcachefs_format.h
View File

@ -578,7 +578,7 @@ struct bch_extent_ptr {
__u64 type:1,
cached:1,
unused:1,
reservation:1,
unwritten:1,
offset:44, /* 8 petabytes */
dev:8,
gen:8;
@ -586,7 +586,7 @@ struct bch_extent_ptr {
__u64 gen:8,
dev:8,
offset:44,
reservation:1,
unwritten:1,
unused:1,
cached:1,
type:1;
@ -794,7 +794,8 @@ struct bch_inode_generation {
x(bi_dir, 64) \
x(bi_dir_offset, 64) \
x(bi_subvol, 32) \
x(bi_parent_subvol, 32)
x(bi_parent_subvol, 32) \
x(bi_nocow, 8)
/* subset of BCH_INODE_FIELDS */
#define BCH_INODE_OPTS() \
@ -806,7 +807,8 @@ struct bch_inode_generation {
x(promote_target, 16) \
x(foreground_target, 16) \
x(background_target, 16) \
x(erasure_code, 16)
x(erasure_code, 16) \
x(nocow, 8)
enum inode_opt_id {
#define x(name, ...) \
@ -1543,7 +1545,8 @@ struct bch_sb_field_journal_seq_blacklist {
x(alloc_v4, 20) \
x(new_data_types, 21) \
x(backpointers, 22) \
x(inode_v3, 23)
x(inode_v3, 23) \
x(unwritten_extents, 24)
enum bcachefs_metadata_version {
bcachefs_metadata_version_min = 9,
@ -1692,6 +1695,7 @@ LE64_BITMASK(BCH_SB_JOURNAL_FLUSH_DISABLED,struct bch_sb, flags[3], 62, 63);
LE64_BITMASK(BCH_SB_JOURNAL_RECLAIM_DELAY,struct bch_sb, flags[4], 0, 32);
/* Obsolete, always enabled: */
LE64_BITMASK(BCH_SB_JOURNAL_TRANSACTION_NAMES,struct bch_sb, flags[4], 32, 33);
LE64_BITMASK(BCH_SB_NOCOW, struct bch_sb, flags[4], 33, 34);
/*
* Features:

2
libbcachefs/bkey_sort.c
View File

@ -144,6 +144,8 @@ bch2_sort_repack(struct bset *dst, struct btree *src,
else
bch2_bkey_unpack(src, (void *) out, in);
out->needs_whiteout = false;
btree_keys_account_key_add(&nr, 0, out);
out = bkey_next(out);
}

5
libbcachefs/bset.h
View File

@ -432,6 +432,11 @@ struct bkey_s_c bch2_btree_node_iter_peek_unpack(struct btree_node_iter *,
struct btree *,
struct bkey *);
#define for_each_btree_node_key(b, k, iter) \
for (bch2_btree_node_iter_init_from_start((iter), (b)); \
(k = bch2_btree_node_iter_peek((iter), (b))); \
bch2_btree_node_iter_advance(iter, b))
#define for_each_btree_node_key_unpack(b, k, iter, unpacked) \
for (bch2_btree_node_iter_init_from_start((iter), (b)); \
(k = bch2_btree_node_iter_peek_unpack((iter), (b), (unpacked))).k;\

1
libbcachefs/btree_cache.c
View File

@ -677,7 +677,6 @@ out:
b->flags = 0;
b->written = 0;
b->nsets = 0;
b->write_type = 0;
b->sib_u64s[0] = 0;
b->sib_u64s[1] = 0;
b->whiteout_u64s = 0;

17
libbcachefs/btree_io.c
View File

@ -1641,6 +1641,7 @@ static void __btree_node_write_done(struct bch_fs *c, struct btree *b)
{
struct btree_write *w = btree_prev_write(b);
unsigned long old, new, v;
unsigned type = 0;
bch2_btree_complete_write(c, b, w);
@ -1659,6 +1660,9 @@ static void __btree_node_write_done(struct bch_fs *c, struct btree *b)
new |= (1U << BTREE_NODE_write_in_flight_inner);
new |= (1U << BTREE_NODE_just_written);
new ^= (1U << BTREE_NODE_write_idx);
type = new & BTREE_WRITE_TYPE_MASK;
new &= ~BTREE_WRITE_TYPE_MASK;
} else {
new &= ~(1U << BTREE_NODE_write_in_flight);
new &= ~(1U << BTREE_NODE_write_in_flight_inner);
@ -1666,7 +1670,7 @@ static void __btree_node_write_done(struct bch_fs *c, struct btree *b)
} while ((v = cmpxchg(&b->flags, old, new)) != old);
if (new & (1U << BTREE_NODE_write_in_flight))
__bch2_btree_node_write(c, b, BTREE_WRITE_ALREADY_STARTED|b->write_type);
__bch2_btree_node_write(c, b, BTREE_WRITE_ALREADY_STARTED|type);
else
wake_up_bit(&b->flags, BTREE_NODE_write_in_flight);
}
@ -1798,7 +1802,8 @@ static void btree_write_submit(struct work_struct *work)
bkey_for_each_ptr(bch2_bkey_ptrs(bkey_i_to_s(&tmp.k)), ptr)
ptr->offset += wbio->sector_offset;
bch2_submit_wbio_replicas(&wbio->wbio, wbio->wbio.c, BCH_DATA_btree, &tmp.k);
bch2_submit_wbio_replicas(&wbio->wbio, wbio->wbio.c, BCH_DATA_btree,
&tmp.k, false);
}
void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, unsigned flags)
@ -1851,6 +1856,10 @@ void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, unsigned flags)
if (old & (1 << BTREE_NODE_write_in_flight))
return;
if (flags & BTREE_WRITE_ONLY_IF_NEED)
type = new & BTREE_WRITE_TYPE_MASK;
new &= ~BTREE_WRITE_TYPE_MASK;
new &= ~(1 << BTREE_NODE_dirty);
new &= ~(1 << BTREE_NODE_need_write);
new |= (1 << BTREE_NODE_write_in_flight);
@ -1862,10 +1871,6 @@ void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, unsigned flags)
if (new & (1U << BTREE_NODE_need_write))
return;
do_write:
if ((flags & BTREE_WRITE_ONLY_IF_NEED))
type = b->write_type;
b->write_type = 0;
BUG_ON((type == BTREE_WRITE_initial) != (b->written == 0));
atomic_dec(&c->btree_cache.dirty);

2
libbcachefs/btree_iter.c
View File

@ -1091,7 +1091,7 @@ static int btree_path_traverse_one(struct btree_trans *trans,
unsigned long trace_ip)
{
unsigned depth_want = path->level;
int ret = trans->restarted;
int ret = -((int) trans->restarted);
if (unlikely(ret))
goto out;

11
libbcachefs/btree_iter.h
View File

@ -347,6 +347,11 @@ __btree_iter_peek_node_and_restart(struct btree_trans *trans, struct btree_iter
return b;
}
/*
* XXX
* this does not handle transaction restarts from bch2_btree_iter_next_node()
* correctly
*/
#define __for_each_btree_node(_trans, _iter, _btree_id, _start, \
_locks_want, _depth, _flags, _b, _ret) \
for (bch2_trans_node_iter_init((_trans), &(_iter), (_btree_id), \
@ -473,11 +478,11 @@ __bch2_btree_iter_peek_and_restart(struct btree_trans *trans,
\
while (1) { \
u32 _restart_count = bch2_trans_begin(_trans); \
\
_ret = 0; \
(_k) = bch2_btree_iter_peek_type(&(_iter), (_flags)); \
if (!(_k).k) { \
_ret = 0; \
if (!(_k).k) \
break; \
} \
\
_ret = bkey_err(_k) ?: (_do); \
if (bch2_err_matches(_ret, BCH_ERR_transaction_restart))\

11
libbcachefs/btree_key_cache.c
View File

@ -190,7 +190,8 @@ static void bkey_cached_free_fast(struct btree_key_cache *bc,
}
static struct bkey_cached *
bkey_cached_alloc(struct btree_trans *trans, struct btree_path *path)
bkey_cached_alloc(struct btree_trans *trans, struct btree_path *path,
bool *was_new)
{
struct bch_fs *c = trans->c;
struct btree_key_cache *bc = &c->btree_key_cache;
@ -274,6 +275,7 @@ bkey_cached_alloc(struct btree_trans *trans, struct btree_path *path)
ck->c.cached = true;
BUG_ON(!six_trylock_intent(&ck->c.lock));
BUG_ON(!six_trylock_write(&ck->c.lock));
*was_new = true;
return ck;
}
@ -312,9 +314,9 @@ btree_key_cache_create(struct btree_trans *trans, struct btree_path *path)
struct bch_fs *c = trans->c;
struct btree_key_cache *bc = &c->btree_key_cache;
struct bkey_cached *ck;
bool was_new = true;
bool was_new = false;
ck = bkey_cached_alloc(trans, path);
ck = bkey_cached_alloc(trans, path, &was_new);
if (IS_ERR(ck))
return ck;
@ -327,7 +329,6 @@ btree_key_cache_create(struct btree_trans *trans, struct btree_path *path)
}
mark_btree_node_locked(trans, path, 0, SIX_LOCK_intent);
was_new = false;
} else {
if (path->btree_id == BTREE_ID_subvolumes)
six_lock_pcpu_alloc(&ck->c.lock);
@ -348,12 +349,12 @@ btree_key_cache_create(struct btree_trans *trans, struct btree_path *path)
if (likely(was_new)) {
six_unlock_write(&ck->c.lock);
six_unlock_intent(&ck->c.lock);
mark_btree_node_locked(trans, path, 0, BTREE_NODE_UNLOCKED);
kfree(ck);
} else {
bkey_cached_free_fast(bc, ck);
}
mark_btree_node_locked(trans, path, 0, BTREE_NODE_UNLOCKED);
return NULL;
}

20
libbcachefs/btree_types.h
View File

@ -77,7 +77,6 @@ struct btree {
u8 nsets;
u8 nr_key_bits;
u16 version_ondisk;
u8 write_type;
struct bkey_format format;
@ -457,6 +456,23 @@ struct btree_trans {
struct replicas_delta_list *fs_usage_deltas;
};
#define BCH_BTREE_WRITE_TYPES() \
x(initial, 0) \
x(init_next_bset, 1) \
x(cache_reclaim, 2) \
x(journal_reclaim, 3) \
x(interior, 4)
enum btree_write_type {
#define x(t, n) BTREE_WRITE_##t,
BCH_BTREE_WRITE_TYPES()
#undef x
BTREE_WRITE_TYPE_NR,
};
#define BTREE_WRITE_TYPE_MASK (roundup_pow_of_two(BTREE_WRITE_TYPE_NR) - 1)
#define BTREE_WRITE_TYPE_BITS ilog2(roundup_pow_of_two(BTREE_WRITE_TYPE_NR))
#define BTREE_FLAGS() \
x(read_in_flight) \
x(read_error) \
@ -476,6 +492,8 @@ struct btree_trans {
x(never_write)
enum btree_flags {
/* First bits for btree node write type */
BTREE_NODE_FLAGS_START = BTREE_WRITE_TYPE_BITS - 1,
#define x(flag) BTREE_NODE_##flag,
BTREE_FLAGS()
#undef x

270
libbcachefs/btree_update_interior.c
View File

@ -378,14 +378,19 @@ static void btree_set_max(struct btree *b, struct bpos pos)
b->data->max_key = pos;
}
struct btree *__bch2_btree_node_alloc_replacement(struct btree_update *as,
struct btree_trans *trans,
struct btree *b,
struct bkey_format format)
static struct btree *bch2_btree_node_alloc_replacement(struct btree_update *as,
struct btree_trans *trans,
struct btree *b)
{
struct btree *n;
struct btree *n = bch2_btree_node_alloc(as, trans, b->c.level);
struct bkey_format format = bch2_btree_calc_format(b);
n = bch2_btree_node_alloc(as, trans, b->c.level);
/*
* The keys might expand with the new format - if they wouldn't fit in
* the btree node anymore, use the old format for now:
*/
if (!bch2_btree_node_format_fits(as->c, b, &format))
format = b->format;
SET_BTREE_NODE_SEQ(n->data, BTREE_NODE_SEQ(b->data) + 1);
@ -398,27 +403,9 @@ struct btree *__bch2_btree_node_alloc_replacement(struct btree_update *as,
bch2_btree_sort_into(as->c, n, b);
btree_node_reset_sib_u64s(n);
n->key.k.p = b->key.k.p;
return n;
}
static struct btree *bch2_btree_node_alloc_replacement(struct btree_update *as,
struct btree_trans *trans,
struct btree *b)
{
struct bkey_format new_f = bch2_btree_calc_format(b);
/*
* The keys might expand with the new format - if they wouldn't fit in
* the btree node anymore, use the old format for now:
*/
if (!bch2_btree_node_format_fits(as->c, b, &new_f))
new_f = b->format;
return __bch2_btree_node_alloc_replacement(as, trans, b, new_f);
}
static struct btree *__btree_root_alloc(struct btree_update *as,
struct btree_trans *trans, unsigned level)
{
@ -1271,6 +1258,7 @@ static void bch2_insert_fixup_btree_ptr(struct btree_update *as,
struct bch_fs *c = as->c;
struct bkey_packed *k;
struct printbuf buf = PRINTBUF;
unsigned long old, new, v;
BUG_ON(insert->k.type == KEY_TYPE_btree_ptr_v2 &&
!btree_ptr_sectors_written(insert));
@ -1308,8 +1296,15 @@ static void bch2_insert_fixup_btree_ptr(struct btree_update *as,
bch2_btree_bset_insert_key(trans, path, b, node_iter, insert);
set_btree_node_dirty_acct(c, b);
set_btree_node_need_write(b);
b->write_type = BTREE_WRITE_interior;
v = READ_ONCE(b->flags);
do {
old = new = v;
new &= ~BTREE_WRITE_TYPE_MASK;
new |= BTREE_WRITE_interior;
new |= 1 << BTREE_NODE_need_write;
} while ((v = cmpxchg(&b->flags, old, new)) != old);
printbuf_exit(&buf);
}
@ -1332,8 +1327,12 @@ __bch2_btree_insert_keys_interior(struct btree_update *as,
;
while (!bch2_keylist_empty(keys)) {
bch2_insert_fixup_btree_ptr(as, trans, path, b,
&node_iter, bch2_keylist_front(keys));
struct bkey_i *k = bch2_keylist_front(keys);
if (bpos_cmp(k->k.p, b->key.k.p) > 0)
break;
bch2_insert_fixup_btree_ptr(as, trans, path, b, &node_iter, k);
bch2_keylist_pop_front(keys);
}
}
@ -1342,109 +1341,91 @@ __bch2_btree_insert_keys_interior(struct btree_update *as,
* Move keys from n1 (original replacement node, now lower node) to n2 (higher
* node)
*/
static struct btree *__btree_split_node(struct btree_update *as,
struct btree_trans *trans,
struct btree *n1)
static void __btree_split_node(struct btree_update *as,
struct btree_trans *trans,
struct btree *b,
struct btree *n[2])
{
struct bkey_format_state s;
size_t nr_packed = 0, nr_unpacked = 0;
struct btree *n2;
struct bset *set1, *set2;
struct bkey_packed *k, *set2_start, *set2_end, *out, *prev = NULL;
struct bkey_packed *k;
struct bpos n1_pos;
struct btree_node_iter iter;
struct bset *bsets[2];
struct bkey_format_state format[2];
struct bkey_packed *out[2];
struct bkey uk;
unsigned u64s, n1_u64s = (b->nr.live_u64s * 3) / 5;
int i;
n2 = bch2_btree_node_alloc(as, trans, n1->c.level);
for (i = 0; i < 2; i++) {
BUG_ON(n[i]->nsets != 1);
n2->data->max_key = n1->data->max_key;
n2->data->format = n1->format;
SET_BTREE_NODE_SEQ(n2->data, BTREE_NODE_SEQ(n1->data));
n2->key.k.p = n1->key.k.p;
bsets[i] = btree_bset_first(n[i]);
out[i] = bsets[i]->start;
set1 = btree_bset_first(n1);
set2 = btree_bset_first(n2);
SET_BTREE_NODE_SEQ(n[i]->data, BTREE_NODE_SEQ(b->data) + 1);
bch2_bkey_format_init(&format[i]);
}
/*
* Has to be a linear search because we don't have an auxiliary
* search tree yet
*/
k = set1->start;
while (1) {
struct bkey_packed *n = bkey_next(k);
u64s = 0;
for_each_btree_node_key(b, k, &iter) {
if (bkey_deleted(k))
continue;
if (n == vstruct_last(set1))
break;
if (k->_data - set1->_data >= (le16_to_cpu(set1->u64s) * 3) / 5)
break;
i = u64s >= n1_u64s;
u64s += k->u64s;
uk = bkey_unpack_key(b, k);
if (!i)
n1_pos = uk.p;
bch2_bkey_format_add_key(&format[i], &uk);
}
if (bkey_packed(k))
nr_packed++;
btree_set_min(n[0], b->data->min_key);
btree_set_max(n[0], n1_pos);
btree_set_min(n[1], bpos_successor(n1_pos));
btree_set_max(n[1], b->data->max_key);
for (i = 0; i < 2; i++) {
bch2_bkey_format_add_pos(&format[i], n[i]->data->min_key);
bch2_bkey_format_add_pos(&format[i], n[i]->data->max_key);
n[i]->data->format = bch2_bkey_format_done(&format[i]);
btree_node_set_format(n[i], n[i]->data->format);
}
u64s = 0;
for_each_btree_node_key(b, k, &iter) {
if (bkey_deleted(k))
continue;
i = u64s >= n1_u64s;
u64s += k->u64s;
if (bch2_bkey_transform(&n[i]->format, out[i], bkey_packed(k)
? &b->format: &bch2_bkey_format_current, k))
out[i]->format = KEY_FORMAT_LOCAL_BTREE;
else
nr_unpacked++;
bch2_bkey_unpack(b, (void *) out[i], k);
prev = k;
k = n;
out[i]->needs_whiteout = false;
btree_keys_account_key_add(&n[i]->nr, 0, out[i]);
out[i] = bkey_next(out[i]);
}
BUG_ON(!prev);
set2_start = k;
set2_end = vstruct_last(set1);
for (i = 0; i < 2; i++) {
bsets[i]->u64s = cpu_to_le16((u64 *) out[i] - bsets[i]->_data);
set1->u64s = cpu_to_le16((u64 *) set2_start - set1->_data);
set_btree_bset_end(n1, n1->set);
BUG_ON(!bsets[i]->u64s);
n1->nr.live_u64s = le16_to_cpu(set1->u64s);
n1->nr.bset_u64s[0] = le16_to_cpu(set1->u64s);
n1->nr.packed_keys = nr_packed;
n1->nr.unpacked_keys = nr_unpacked;
set_btree_bset_end(n[i], n[i]->set);
n1_pos = bkey_unpack_pos(n1, prev);
if (as->c->sb.version < bcachefs_metadata_version_snapshot)
n1_pos.snapshot = U32_MAX;
btree_node_reset_sib_u64s(n[i]);
btree_set_max(n1, n1_pos);
btree_set_min(n2, bpos_successor(n1->key.k.p));
bch2_verify_btree_nr_keys(n[i]);
bch2_bkey_format_init(&s);
bch2_bkey_format_add_pos(&s, n2->data->min_key);
bch2_bkey_format_add_pos(&s, n2->data->max_key);
for (k = set2_start; k != set2_end; k = bkey_next(k)) {
struct bkey uk = bkey_unpack_key(n1, k);
bch2_bkey_format_add_key(&s, &uk);
if (b->c.level)
btree_node_interior_verify(as->c, n[i]);
}
n2->data->format = bch2_bkey_format_done(&s);
btree_node_set_format(n2, n2->data->format);
out = set2->start;
memset(&n2->nr, 0, sizeof(n2->nr));
for (k = set2_start; k != set2_end; k = bkey_next(k)) {
BUG_ON(!bch2_bkey_transform(&n2->format, out, bkey_packed(k)
? &n1->format : &bch2_bkey_format_current, k));
out->format = KEY_FORMAT_LOCAL_BTREE;
btree_keys_account_key_add(&n2->nr, 0, out);
out = bkey_next(out);
}
set2->u64s = cpu_to_le16((u64 *) out - set2->_data);
set_btree_bset_end(n2, n2->set);
BUG_ON(!set1->u64s);
BUG_ON(!set2->u64s);
btree_node_reset_sib_u64s(n1);
btree_node_reset_sib_u64s(n2);
bch2_verify_btree_nr_keys(n1);
bch2_verify_btree_nr_keys(n2);
if (n1->c.level) {
btree_node_interior_verify(as->c, n1);
btree_node_interior_verify(as->c, n2);
}
return n2;
}
/*
@ -1464,41 +1445,17 @@ static void btree_split_insert_keys(struct btree_update *as,
struct btree *b,
struct keylist *keys)
{
struct btree_node_iter node_iter;
struct bkey_i *k = bch2_keylist_front(keys);
struct bkey_packed *src, *dst, *n;
struct bset *i;
if (!bch2_keylist_empty(keys) &&
bpos_cmp(bch2_keylist_front(keys)->k.p,
b->data->max_key) <= 0) {
struct btree_node_iter node_iter;
bch2_btree_node_iter_init(&node_iter, b, &k->k.p);
bch2_btree_node_iter_init(&node_iter, b, &bch2_keylist_front(keys)->k.p);
__bch2_btree_insert_keys_interior(as, trans, path, b, node_iter, keys);
__bch2_btree_insert_keys_interior(as, trans, path, b, node_iter, keys);
/*
* We can't tolerate whiteouts here - with whiteouts there can be
* duplicate keys, and it would be rather bad if we picked a duplicate
* for the pivot:
*/
i = btree_bset_first(b);
src = dst = i->start;
while (src != vstruct_last(i)) {
n = bkey_next(src);
if (!bkey_deleted(src)) {
memmove_u64s_down(dst, src, src->u64s);
dst = bkey_next(dst);
}
src = n;
btree_node_interior_verify(as->c, b);
}
/* Also clear out the unwritten whiteouts area: */
b->whiteout_u64s = 0;
i->u64s = cpu_to_le16((u64 *) dst - i->_data);
set_btree_bset_end(b, b->set);
BUG_ON(b->nsets != 1 ||
b->nr.live_u64s != le16_to_cpu(btree_bset_first(b)->u64s));
btree_node_interior_verify(as->c, b);
}
static int btree_split(struct btree_update *as, struct btree_trans *trans,
@ -1517,15 +1474,21 @@ static int btree_split(struct btree_update *as, struct btree_trans *trans,
bch2_btree_interior_update_will_free_node(as, b);
n1 = bch2_btree_node_alloc_replacement(as, trans, b);
if (b->nr.live_u64s > BTREE_SPLIT_THRESHOLD(c)) {
struct btree *n[2];
if (keys)
btree_split_insert_keys(as, trans, path, n1, keys);
if (bset_u64s(&n1->set[0]) > BTREE_SPLIT_THRESHOLD(c)) {
trace_and_count(c, btree_node_split, c, b);
n2 = __btree_split_node(as, trans, n1);
n[0] = n1 = bch2_btree_node_alloc(as, trans, b->c.level);
n[1] = n2 = bch2_btree_node_alloc(as, trans, b->c.level);
__btree_split_node(as, trans, b, n);
if (keys) {
btree_split_insert_keys(as, trans, path, n1, keys);
btree_split_insert_keys(as, trans, path, n2, keys);
BUG_ON(!bch2_keylist_empty(keys));
}
bch2_btree_build_aux_trees(n2);
bch2_btree_build_aux_trees(n1);
@ -1574,6 +1537,13 @@ static int btree_split(struct btree_update *as, struct btree_trans *trans,
} else {
trace_and_count(c, btree_node_compact, c, b);
n1 = bch2_btree_node_alloc_replacement(as, trans, b);
if (keys) {
btree_split_insert_keys(as, trans, path, n1, keys);
BUG_ON(!bch2_keylist_empty(keys));
}
bch2_btree_build_aux_trees(n1);
bch2_btree_update_add_new_node(as, n1);
six_unlock_write(&n1->c.lock);

4
libbcachefs/btree_update_leaf.c
View File

@ -178,11 +178,11 @@ static int __btree_node_flush(struct journal *j, struct journal_entry_pin *pin,
w->journal.seq != seq)
break;
new &= ~BTREE_WRITE_TYPE_MASK;
new |= BTREE_WRITE_journal_reclaim;
new |= 1 << BTREE_NODE_need_write;
} while ((v = cmpxchg(&b->flags, old, new)) != old);
b->write_type = BTREE_WRITE_journal_reclaim;
btree_node_write_if_need(c, b, SIX_LOCK_read);
six_unlock_read(&b->c.lock);

3
libbcachefs/buckets_waiting_for_journal.c
View File

@ -3,6 +3,7 @@
#include "bcachefs.h"
#include "buckets_waiting_for_journal.h"
#include <linux/random.h>
#include <linux/siphash.h>
static inline struct bucket_hashed *
bucket_hash(struct buckets_waiting_for_journal_table *t,
@ -10,7 +11,7 @@ bucket_hash(struct buckets_waiting_for_journal_table *t,
{
unsigned h = siphash_1u64(dev_bucket, &t->hash_seeds[hash_seed_idx]);
BUG_ON(!is_power_of_2(t->size));
EBUG_ON(!is_power_of_2(t->size));
return t->d + (h & (t->size - 1));
}

7
libbcachefs/checksum.h
View File

@ -99,14 +99,17 @@ static inline enum bch_csum_type bch2_csum_opt_to_type(enum bch_csum_opts type,
}
static inline enum bch_csum_type bch2_data_checksum_type(struct bch_fs *c,
unsigned opt)
struct bch_io_opts opts)
{
if (opts.nocow)
return 0;
if (c->sb.encryption_type)
return c->opts.wide_macs
? BCH_CSUM_chacha20_poly1305_128
: BCH_CSUM_chacha20_poly1305_80;
return bch2_csum_opt_to_type(opt, true);
return bch2_csum_opt_to_type(opts.data_checksum, true);
}
static inline enum bch_csum_type bch2_meta_checksum_type(struct bch_fs *c)

128
libbcachefs/data_update.c
View File

@ -97,10 +97,10 @@ static void bch2_bkey_mark_dev_cached(struct bkey_s k, unsigned dev)
ptr->cached = true;
}
int bch2_data_update_index_update(struct bch_write_op *op)
static int __bch2_data_update_index_update(struct btree_trans *trans,
struct bch_write_op *op)
{
struct bch_fs *c = op->c;
struct btree_trans trans;
struct btree_iter iter;
struct data_update *m =
container_of(op, struct data_update, op);
@ -112,9 +112,7 @@ int bch2_data_update_index_update(struct bch_write_op *op)
bch2_bkey_buf_init(&_insert);
bch2_bkey_buf_realloc(&_insert, c, U8_MAX);
bch2_trans_init(&trans, c, BTREE_ITER_MAX, 1024);
bch2_trans_iter_init(&trans, &iter, m->btree_id,
bch2_trans_iter_init(trans, &iter, m->btree_id,
bkey_start_pos(&bch2_keylist_front(keys)->k),
BTREE_ITER_SLOTS|BTREE_ITER_INTENT);
@ -131,7 +129,7 @@ int bch2_data_update_index_update(struct bch_write_op *op)
s64 i_sectors_delta = 0, disk_sectors_delta = 0;
unsigned i;
bch2_trans_begin(&trans);
bch2_trans_begin(trans);
k = bch2_btree_iter_peek_slot(&iter);
ret = bkey_err(k);
@ -202,7 +200,7 @@ int bch2_data_update_index_update(struct bch_write_op *op)
bch2_bkey_narrow_crcs(insert, (struct bch_extent_crc_unpacked) { 0 });
bch2_extent_normalize(c, bkey_i_to_s(insert));
ret = bch2_sum_sector_overwrites(&trans, &iter, insert,
ret = bch2_sum_sector_overwrites(trans, &iter, insert,
&should_check_enospc,
&i_sectors_delta,
&disk_sectors_delta);
@ -220,11 +218,11 @@ int bch2_data_update_index_update(struct bch_write_op *op)
next_pos = insert->k.p;
ret = insert_snapshot_whiteouts(&trans, m->btree_id,
ret = insert_snapshot_whiteouts(trans, m->btree_id,
k.k->p, insert->k.p) ?:
bch2_trans_update(&trans, &iter, insert,
bch2_trans_update(trans, &iter, insert,
BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) ?:
bch2_trans_commit(&trans, &op->res,
bch2_trans_commit(trans, &op->res,
NULL,
BTREE_INSERT_NOFAIL|
m->data_opts.btree_insert_flags);
@ -261,14 +259,26 @@ nomatch:
goto next;
}
out:
bch2_trans_iter_exit(&trans, &iter);
bch2_trans_exit(&trans);
bch2_trans_iter_exit(trans, &iter);
bch2_bkey_buf_exit(&_insert, c);
bch2_bkey_buf_exit(&_new, c);
BUG_ON(bch2_err_matches(ret, BCH_ERR_transaction_restart));
return ret;
}
int bch2_data_update_index_update(struct bch_write_op *op)
{
struct bch_fs *c = op->c;
struct btree_trans trans;
int ret;
bch2_trans_init(&trans, c, BTREE_ITER_MAX, 1024);
ret = __bch2_data_update_index_update(&trans, op);
bch2_trans_exit(&trans);
return ret;
}
void bch2_data_update_read_done(struct data_update *m,
struct bch_extent_crc_unpacked crc)
{
@ -284,12 +294,99 @@ void bch2_data_update_read_done(struct data_update *m,
void bch2_data_update_exit(struct data_update *update)
{
struct bch_fs *c = update->op.c;
struct bkey_ptrs_c ptrs =
bch2_bkey_ptrs_c(bkey_i_to_s_c(update->k.k));
const struct bch_extent_ptr *ptr;
bkey_for_each_ptr(ptrs, ptr)
bch2_bucket_nocow_unlock(&c->nocow_locks,
PTR_BUCKET_POS(c, ptr), 0);
bch2_bkey_buf_exit(&update->k, c);
bch2_disk_reservation_put(c, &update->op.res);
bch2_bio_free_pages_pool(c, &update->op.wbio.bio);
}
void bch2_update_unwritten_extent(struct btree_trans *trans,
struct data_update *update)
{
struct bch_fs *c = update->op.c;
struct bio *bio = &update->op.wbio.bio;
struct bkey_i_extent *e;
struct write_point *wp;
struct bch_extent_ptr *ptr;
struct closure cl;
struct btree_iter iter;
struct bkey_s_c k;
int ret;
closure_init_stack(&cl);
bch2_keylist_init(&update->op.insert_keys, update->op.inline_keys);
while (bio_sectors(bio)) {
unsigned sectors = bio_sectors(bio);
bch2_trans_iter_init(trans, &iter, update->btree_id, update->op.pos,
BTREE_ITER_SLOTS);
ret = lockrestart_do(trans, ({
k = bch2_btree_iter_peek_slot(&iter);
bkey_err(k);
}));
bch2_trans_iter_exit(trans, &iter);
if (ret || !bch2_extents_match(k, bkey_i_to_s_c(update->k.k)))
break;
e = bkey_extent_init(update->op.insert_keys.top);
e->k.p = update->op.pos;
ret = bch2_alloc_sectors_start_trans(trans,
update->op.target,
false,
update->op.write_point,
&update->op.devs_have,
update->op.nr_replicas,
update->op.nr_replicas,
update->op.alloc_reserve,
0, &cl, &wp);
if (ret == -EAGAIN) {
bch2_trans_unlock(trans);
closure_sync(&cl);
continue;
}
if (ret)
return;
sectors = min(sectors, wp->sectors_free);
bch2_key_resize(&e->k, sectors);
bch2_open_bucket_get(c, wp, &update->op.open_buckets);
bch2_alloc_sectors_append_ptrs(c, wp, &e->k_i, sectors, false);
bch2_alloc_sectors_done(c, wp);
bio_advance(bio, sectors << 9);
update->op.pos.offset += sectors;
extent_for_each_ptr(extent_i_to_s(e), ptr)
ptr->unwritten = true;
bch2_keylist_push(&update->op.insert_keys);
ret = __bch2_data_update_index_update(trans, &update->op);
bch2_open_buckets_put(c, &update->op.open_buckets);
if (ret)
break;
}
if ((atomic_read(&cl.remaining) & CLOSURE_REMAINING_MASK) != 1) {
bch2_trans_unlock(trans);
closure_sync(&cl);
}
}
int bch2_data_update_init(struct bch_fs *c, struct data_update *m,
struct write_point_specifier wp,
struct bch_io_opts io_opts,
@ -351,6 +448,9 @@ int bch2_data_update_init(struct bch_fs *c, struct data_update *m,
m->op.incompressible = true;
i++;
bch2_bucket_nocow_lock(&c->nocow_locks,
PTR_BUCKET_POS(c, &p.ptr), 0);
}
if (reserve_sectors) {
@ -366,6 +466,10 @@ int bch2_data_update_init(struct bch_fs *c, struct data_update *m,
hweight32(m->data_opts.rewrite_ptrs) + m->data_opts.extra_replicas;
BUG_ON(!m->op.nr_replicas);
/* Special handling required: */
if (bkey_extent_is_unwritten(k))
return -BCH_ERR_unwritten_extent_update;
return 0;
}

1
libbcachefs/data_update.h
View File

@ -32,6 +32,7 @@ void bch2_data_update_read_done(struct data_update *,
struct bch_extent_crc_unpacked);
void bch2_data_update_exit(struct data_update *);
void bch2_update_unwritten_extent(struct btree_trans *, struct data_update *);
int bch2_data_update_init(struct bch_fs *, struct data_update *,
struct write_point_specifier,
struct bch_io_opts, struct data_update_opts,

3
libbcachefs/errcode.h
View File

@ -63,7 +63,8 @@
x(BCH_ERR_fsck, fsck_repair_unimplemented) \
x(BCH_ERR_fsck, fsck_repair_impossible) \
x(0, need_snapshot_cleanup) \
x(0, need_topology_repair)
x(0, need_topology_repair) \
x(0, unwritten_extent_update)
enum bch_errcode {
BCH_ERR_START = 2048,

29
libbcachefs/error.h
View File

@ -186,36 +186,25 @@ void bch2_io_error_work(struct work_struct *);
/* Does the error handling without logging a message */
void bch2_io_error(struct bch_dev *);
/* Logs message and handles the error: */
#define bch2_dev_io_error(ca, fmt, ...) \
do { \
printk_ratelimited(KERN_ERR "bcachefs (%s): " fmt, \
(ca)->name, ##__VA_ARGS__); \
bch2_io_error(ca); \
} while (0)
#define bch2_dev_inum_io_error(ca, _inum, _offset, fmt, ...) \
do { \
printk_ratelimited(KERN_ERR "bcachefs (%s inum %llu offset %llu): " fmt,\
(ca)->name, (_inum), (_offset), ##__VA_ARGS__); \
bch2_io_error(ca); \
} while (0)
#define bch2_dev_io_err_on(cond, ca, ...) \
({ \
bool _ret = (cond); \
\
if (_ret) \
bch2_dev_io_error(ca, __VA_ARGS__); \
if (_ret) { \
bch_err_dev_ratelimited(ca, __VA_ARGS__); \
bch2_io_error(ca); \
} \
_ret; \
})
#define bch2_dev_inum_io_err_on(cond, ca, _inum, _offset, ...) \
#define bch2_dev_inum_io_err_on(cond, ca, ...) \
({ \
bool _ret = (cond); \
\
if (_ret) \
bch2_dev_inum_io_error(ca, _inum, _offset, __VA_ARGS__);\
if (_ret) { \
bch_err_inum_offset_ratelimited(ca, __VA_ARGS__); \
bch2_io_error(ca); \
} \
_ret; \
})

69
libbcachefs/extents.c
View File

@ -117,6 +117,13 @@ int bch2_bkey_pick_read_device(struct bch_fs *c, struct bkey_s_c k,
return -EIO;
bkey_for_each_ptr_decode(k.k, ptrs, p, entry) {
/*
* Unwritten extent: no need to actually read, treat it as a
* hole and return 0s:
*/
if (p.ptr.unwritten)
return 0;
ca = bch_dev_bkey_exists(c, p.ptr.dev);
/*
@ -270,6 +277,7 @@ bool bch2_extent_merge(struct bch_fs *c, struct bkey_s l, struct bkey_s_c r)
rp.ptr.offset + rp.crc.offset ||
lp.ptr.dev != rp.ptr.dev ||
lp.ptr.gen != rp.ptr.gen ||
lp.ptr.unwritten != rp.ptr.unwritten ||
lp.has_ec != rp.has_ec)
return false;
@ -657,22 +665,21 @@ unsigned bch2_bkey_replicas(struct bch_fs *c, struct bkey_s_c k)
return replicas;
}
static unsigned bch2_extent_ptr_durability(struct bch_fs *c,
struct extent_ptr_decoded p)
unsigned bch2_extent_ptr_durability(struct bch_fs *c, struct extent_ptr_decoded *p)
{
unsigned durability = 0;
struct bch_dev *ca;
if (p.ptr.cached)
if (p->ptr.cached)
return 0;
ca = bch_dev_bkey_exists(c, p.ptr.dev);
ca = bch_dev_bkey_exists(c, p->ptr.dev);
if (ca->mi.state != BCH_MEMBER_STATE_failed)
durability = max_t(unsigned, durability, ca->mi.durability);
if (p.has_ec)
durability += p.ec.redundancy;
if (p->has_ec)
durability += p->ec.redundancy;
return durability;
}
@ -685,7 +692,7 @@ unsigned bch2_bkey_durability(struct bch_fs *c, struct bkey_s_c k)
unsigned durability = 0;
bkey_for_each_ptr_decode(k.k, ptrs, p, entry)
durability += bch2_extent_ptr_durability(c, p);
durability += bch2_extent_ptr_durability(c,& p);
return durability;
}
@ -923,20 +930,31 @@ bool bch2_bkey_matches_ptr(struct bch_fs *c, struct bkey_s_c k,
*/
bool bch2_extents_match(struct bkey_s_c k1, struct bkey_s_c k2)
{
struct bkey_ptrs_c ptrs1 = bch2_bkey_ptrs_c(k1);
struct bkey_ptrs_c ptrs2 = bch2_bkey_ptrs_c(k2);
const union bch_extent_entry *entry1, *entry2;
struct extent_ptr_decoded p1, p2;
if (k1.k->type != k2.k->type)
return false;
bkey_for_each_ptr_decode(k1.k, ptrs1, p1, entry1)
bkey_for_each_ptr_decode(k2.k, ptrs2, p2, entry2)
if (bkey_extent_is_direct_data(k1.k)) {
struct bkey_ptrs_c ptrs1 = bch2_bkey_ptrs_c(k1);
struct bkey_ptrs_c ptrs2 = bch2_bkey_ptrs_c(k2);
const union bch_extent_entry *entry1, *entry2;
struct extent_ptr_decoded p1, p2;
if (bkey_extent_is_unwritten(k1) != bkey_extent_is_unwritten(k2))
return false;
bkey_for_each_ptr_decode(k1.k, ptrs1, p1, entry1)
bkey_for_each_ptr_decode(k2.k, ptrs2, p2, entry2)
if (p1.ptr.dev == p2.ptr.dev &&
p1.ptr.gen == p2.ptr.gen &&
(s64) p1.ptr.offset + p1.crc.offset - bkey_start_offset(k1.k) ==
(s64) p2.ptr.offset + p2.crc.offset - bkey_start_offset(k2.k))
return true;
return false;
return false;
} else {
/* KEY_TYPE_deleted, etc. */
return true;
}
}
bool bch2_extent_has_ptr(struct bkey_s_c k1, struct extent_ptr_decoded p1,
@ -1005,10 +1023,12 @@ void bch2_bkey_ptrs_to_text(struct printbuf *out, struct bch_fs *c,
u32 offset;
u64 b = sector_to_bucket_and_offset(ca, ptr->offset, &offset);
prt_printf(out, "ptr: %u:%llu:%u gen %u%s", ptr->dev,
b, offset, ptr->gen,
ptr->cached ? " cached" : "");
prt_printf(out, "ptr: %u:%llu:%u gen %u",
ptr->dev, b, offset, ptr->gen);
if (ptr->cached)
prt_str(out, " cached");
if (ptr->unwritten)
prt_str(out, " unwritten");
if (ca && ptr_stale(ca, ptr))
prt_printf(out, " stale");
}
@ -1097,6 +1117,7 @@ int bch2_bkey_ptrs_invalid(const struct bch_fs *c, struct bkey_s_c k,
unsigned size_ondisk = k.k->size;
unsigned nonce = UINT_MAX;
unsigned nr_ptrs = 0;
bool unwritten = false;
int ret;
if (bkey_is_btree_ptr(k.k))
@ -1121,6 +1142,18 @@ int bch2_bkey_ptrs_invalid(const struct bch_fs *c, struct bkey_s_c k,
false, err);
if (ret)
return ret;
if (nr_ptrs && unwritten != entry->ptr.unwritten) {
prt_printf(err, "extent with unwritten and written ptrs");
return -EINVAL;
}
if (k.k->type != KEY_TYPE_extent && entry->ptr.unwritten) {
prt_printf(err, "has unwritten ptrs");
return -EINVAL;
}
unwritten = entry->ptr.unwritten;
nr_ptrs++;
break;
case BCH_EXTENT_ENTRY_crc32:

18
libbcachefs/extents.h
View File

@ -510,6 +510,23 @@ static inline bool bkey_extent_is_allocation(const struct bkey *k)
}
}
static inline bool bkey_extent_is_unwritten(struct bkey_s_c k)
{
struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
const struct bch_extent_ptr *ptr;
bkey_for_each_ptr(ptrs, ptr)
if (ptr->unwritten)
return true;
return false;
}
static inline bool bkey_extent_is_reservation(struct bkey_s_c k)
{
return k.k->type == KEY_TYPE_reservation ||
bkey_extent_is_unwritten(k);
}
static inline struct bch_devs_list bch2_bkey_devs(struct bkey_s_c k)
{
struct bch_devs_list ret = (struct bch_devs_list) { 0 };
@ -579,6 +596,7 @@ bool bch2_bkey_is_incompressible(struct bkey_s_c);
unsigned bch2_bkey_sectors_compressed(struct bkey_s_c);
unsigned bch2_bkey_replicas(struct bch_fs *, struct bkey_s_c);
unsigned bch2_extent_ptr_durability(struct bch_fs *, struct extent_ptr_decoded *);
unsigned bch2_bkey_durability(struct bch_fs *, struct bkey_s_c);
void bch2_bkey_extent_entry_drop(struct bkey_i *, union bch_extent_entry *);

250
libbcachefs/fs-io.c
View File

@ -35,6 +35,72 @@
#include <trace/events/bcachefs.h>
#include <trace/events/writeback.h>
struct nocow_flush {
struct closure *cl;
struct bch_dev *ca;
struct bio bio;
};
static void nocow_flush_endio(struct bio *_bio)
{
struct nocow_flush *bio = container_of(_bio, struct nocow_flush, bio);
closure_put(bio->cl);
percpu_ref_put(&bio->ca->io_ref);
bio_put(&bio->bio);
}
static void bch2_inode_flush_nocow_writes_async(struct bch_fs *c,
struct bch_inode_info *inode,
struct closure *cl)
{
struct nocow_flush *bio;
struct bch_dev *ca;
struct bch_devs_mask devs;
unsigned dev;
dev = find_first_bit(inode->ei_devs_need_flush.d, BCH_SB_MEMBERS_MAX);
if (dev == BCH_SB_MEMBERS_MAX)
return;
devs = inode->ei_devs_need_flush;
memset(&inode->ei_devs_need_flush, 0, sizeof(inode->ei_devs_need_flush));
for_each_set_bit(dev, devs.d, BCH_SB_MEMBERS_MAX) {
rcu_read_lock();
ca = rcu_dereference(c->devs[dev]);
if (ca && !percpu_ref_tryget(&ca->io_ref))
ca = NULL;
rcu_read_unlock();
if (!ca)
continue;
bio = container_of(bio_alloc_bioset(ca->disk_sb.bdev, 0,
REQ_OP_FLUSH,
GFP_KERNEL,
&c->nocow_flush_bioset),
struct nocow_flush, bio);
bio->cl = cl;
bio->ca = ca;
bio->bio.bi_end_io = nocow_flush_endio;
closure_bio_submit(&bio->bio, cl);
}
}
static int bch2_inode_flush_nocow_writes(struct bch_fs *c,
struct bch_inode_info *inode)
{
struct closure cl;
closure_init_stack(&cl);
bch2_inode_flush_nocow_writes_async(c, inode, &cl);
closure_sync(&cl);
return 0;
}
static inline bool bio_full(struct bio *bio, unsigned len)
{
if (bio->bi_vcnt >= bio->bi_max_vecs)
@ -77,6 +143,7 @@ struct dio_write {
struct bch_inode_info *inode;
struct mm_struct *mm;
unsigned loop:1,
extending:1,
sync:1,
flush:1,
free_iov:1;
@ -131,24 +198,29 @@ static noinline int write_invalidate_inode_pages_range(struct address_space *map
#ifdef CONFIG_BCACHEFS_QUOTA
static void bch2_quota_reservation_put(struct bch_fs *c,
struct bch_inode_info *inode,
struct quota_res *res)
static void __bch2_quota_reservation_put(struct bch_fs *c,
struct bch_inode_info *inode,
struct quota_res *res)
{
if (!res->sectors)
return;
mutex_lock(&inode->ei_quota_lock);
BUG_ON(res->sectors > inode->ei_quota_reserved);
bch2_quota_acct(c, inode->ei_qid, Q_SPC,
-((s64) res->sectors), KEY_TYPE_QUOTA_PREALLOC);
inode->ei_quota_reserved -= res->sectors;
mutex_unlock(&inode->ei_quota_lock);
res->sectors = 0;
}
static void bch2_quota_reservation_put(struct bch_fs *c,
struct bch_inode_info *inode,
struct quota_res *res)
{
if (res->sectors) {
mutex_lock(&inode->ei_quota_lock);
__bch2_quota_reservation_put(c, inode, res);
mutex_unlock(&inode->ei_quota_lock);
}
}
static int bch2_quota_reservation_add(struct bch_fs *c,
struct bch_inode_info *inode,
struct quota_res *res,
@ -171,11 +243,13 @@ static int bch2_quota_reservation_add(struct bch_fs *c,
#else
static void __bch2_quota_reservation_put(struct bch_fs *c,
struct bch_inode_info *inode,
struct quota_res *res) {}
static void bch2_quota_reservation_put(struct bch_fs *c,
struct bch_inode_info *inode,
struct quota_res *res)
{
}
struct quota_res *res) {}
static int bch2_quota_reservation_add(struct bch_fs *c,
struct bch_inode_info *inode,
@ -226,13 +300,9 @@ int __must_check bch2_write_inode_size(struct bch_fs *c,
return bch2_write_inode(c, inode, inode_set_size, &s, fields);
}
static void i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode,
static void __i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode,
struct quota_res *quota_res, s64 sectors)
{
if (!sectors)
return;
mutex_lock(&inode->ei_quota_lock);
bch2_fs_inconsistent_on((s64) inode->v.i_blocks + sectors < 0, c,
"inode %lu i_blocks underflow: %llu + %lli < 0 (ondisk %lli)",
inode->v.i_ino, (u64) inode->v.i_blocks, sectors,
@ -250,7 +320,16 @@ static void i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode,
bch2_quota_acct(c, inode->ei_qid, Q_SPC, sectors, KEY_TYPE_QUOTA_WARN);
}
#endif
mutex_unlock(&inode->ei_quota_lock);
}
static void i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode,
struct quota_res *quota_res, s64 sectors)
{
if (sectors) {
mutex_lock(&inode->ei_quota_lock);
__i_sectors_acct(c, inode, quota_res, sectors);
mutex_unlock(&inode->ei_quota_lock);
}
}
/* page state: */
@ -328,11 +407,11 @@ static struct bch_page_state *bch2_page_state_create(struct page *page,
return bch2_page_state(page) ?: __bch2_page_state_create(page, gfp);
}
static unsigned bkey_to_sector_state(const struct bkey *k)
static unsigned bkey_to_sector_state(struct bkey_s_c k)
{
if (k->type == KEY_TYPE_reservation)
if (bkey_extent_is_reservation(k))
return SECTOR_RESERVED;
if (bkey_extent_is_allocation(k))
if (bkey_extent_is_allocation(k.k))
return SECTOR_ALLOCATED;
return SECTOR_UNALLOCATED;
}
@ -383,7 +462,7 @@ retry:
SPOS(inum.inum, offset, snapshot),
BTREE_ITER_SLOTS, k, ret) {
unsigned nr_ptrs = bch2_bkey_nr_ptrs_fully_allocated(k);
unsigned state = bkey_to_sector_state(k.k);
unsigned state = bkey_to_sector_state(k);
while (pg_idx < nr_pages) {
struct page *page = pages[pg_idx];
@ -423,7 +502,7 @@ static void bch2_bio_page_state_set(struct bio *bio, struct bkey_s_c k)
struct bio_vec bv;
unsigned nr_ptrs = k.k->type == KEY_TYPE_reflink_v
? 0 : bch2_bkey_nr_ptrs_fully_allocated(k);
unsigned state = bkey_to_sector_state(k.k);
unsigned state = bkey_to_sector_state(k);
bio_for_each_segment(bv, bio, iter)
__bch2_page_state_set(bv.bv_page, bv.bv_offset >> 9,
@ -1074,7 +1153,9 @@ err:
goto retry;
if (ret) {
bch_err_inum_ratelimited(c, inum.inum,
bch_err_inum_offset_ratelimited(c,
iter.pos.inode,
iter.pos.offset << 9,
"read error %i from btree lookup", ret);
rbio->bio.bi_status = BLK_STS_IOERR;
bio_endio(&rbio->bio);
@ -1306,6 +1387,7 @@ static void bch2_writepage_io_alloc(struct bch_fs *c,
op->subvol = inode->ei_subvol;
op->pos = POS(inode->v.i_ino, sector);
op->end_io = bch2_writepage_io_done;
op->devs_need_flush = &inode->ei_devs_need_flush;
op->wbio.bio.bi_iter.bi_sector = sector;
op->wbio.bio.bi_opf = wbc_to_write_flags(wbc);
}
@ -1428,9 +1510,13 @@ do_io:
sectors << 9, offset << 9));
/* Check for writing past i_size: */
WARN_ON_ONCE((bio_end_sector(&w->io->op.wbio.bio) << 9) >
round_up(i_size, block_bytes(c)) &&
!test_bit(BCH_FS_EMERGENCY_RO, &c->flags));
WARN_ONCE((bio_end_sector(&w->io->op.wbio.bio) << 9) >
round_up(i_size, block_bytes(c)) &&
!test_bit(BCH_FS_EMERGENCY_RO, &c->flags),
"writing past i_size: %llu > %llu (unrounded %llu)\n",
bio_end_sector(&w->io->op.wbio.bio) << 9,
round_up(i_size, block_bytes(c)),
i_size);
w->io->op.res.sectors += reserved_sectors;
w->io->op.i_sectors_delta -= dirty_sectors;
@ -2101,10 +2187,12 @@ static noinline void bch2_dio_write_flush(struct dio_write *dio)
if (!dio->op.error) {
ret = bch2_inode_find_by_inum(c, inode_inum(dio->inode), &inode);
if (ret)
if (ret) {
dio->op.error = ret;
else
} else {
bch2_journal_flush_seq_async(&c->journal, inode.bi_journal_seq, &dio->op.cl);
bch2_inode_flush_nocow_writes_async(c, dio->inode, &dio->op.cl);
}
}
if (dio->sync) {
@ -2117,7 +2205,6 @@ static noinline void bch2_dio_write_flush(struct dio_write *dio)
static __always_inline long bch2_dio_write_done(struct dio_write *dio)
{
struct bch_fs *c = dio->op.c;
struct kiocb *req = dio->req;
struct bch_inode_info *inode = dio->inode;
bool sync = dio->sync;
@ -2130,7 +2217,6 @@ static __always_inline long bch2_dio_write_done(struct dio_write *dio)
}
bch2_pagecache_block_put(inode);
bch2_quota_reservation_put(c, inode, &dio->quota_res);
if (dio->free_iov)
kfree(dio->iter.iov);
@ -2160,14 +2246,22 @@ static __always_inline void bch2_dio_write_end(struct dio_write *dio)
struct bvec_iter_all iter;
struct bio_vec *bv;
i_sectors_acct(c, inode, &dio->quota_res, dio->op.i_sectors_delta);
req->ki_pos += (u64) dio->op.written << 9;
dio->written += dio->op.written;
req->ki_pos += (u64) dio->op.written << 9;
dio->written += dio->op.written;
spin_lock(&inode->v.i_lock);
if (req->ki_pos > inode->v.i_size)
i_size_write(&inode->v, req->ki_pos);
spin_unlock(&inode->v.i_lock);
if (dio->extending) {
spin_lock(&inode->v.i_lock);
if (req->ki_pos > inode->v.i_size)
i_size_write(&inode->v, req->ki_pos);
spin_unlock(&inode->v.i_lock);
}
if (dio->op.i_sectors_delta || dio->quota_res.sectors) {
mutex_lock(&inode->ei_quota_lock);
__i_sectors_acct(c, inode, &dio->quota_res, dio->op.i_sectors_delta);
__bch2_quota_reservation_put(c, inode, &dio->quota_res);
mutex_unlock(&inode->ei_quota_lock);
}
if (likely(!bio_flagged(bio, BIO_NO_PAGE_REF)))
bio_for_each_segment_all(bv, bio, iter)
@ -2244,11 +2338,17 @@ static long bch2_dio_write_loop(struct dio_write *dio)
dio->op.nr_replicas = dio->op.opts.data_replicas;
dio->op.subvol = inode->ei_subvol;
dio->op.pos = POS(inode->v.i_ino, (u64) req->ki_pos >> 9);
dio->op.devs_need_flush = &inode->ei_devs_need_flush;
if (sync)
dio->op.flags |= BCH_WRITE_SYNC;
dio->op.flags |= BCH_WRITE_CHECK_ENOSPC;
ret = bch2_quota_reservation_add(c, inode, &dio->quota_res,
bio_sectors(bio), true);
if (unlikely(ret))
goto err;
ret = bch2_disk_reservation_get(c, &dio->op.res, bio_sectors(bio),
dio->op.opts.data_replicas, 0);
if (unlikely(ret) &&
@ -2288,6 +2388,8 @@ err:
bio_for_each_segment_all(bv, bio, iter)
put_page(bv->bv_page);
}
bch2_quota_reservation_put(c, inode, &dio->quota_res);
goto out;
}
@ -2366,6 +2468,7 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
dio->inode = inode;
dio->mm = current->mm;
dio->loop = false;
dio->extending = extending;
dio->sync = is_sync_kiocb(req) || extending;
dio->flush = iocb_is_dsync(req) && !c->opts.journal_flush_disabled;
dio->free_iov = false;
@ -2374,11 +2477,6 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
dio->iter = *iter;
dio->op.c = c;
ret = bch2_quota_reservation_add(c, inode, &dio->quota_res,
iter->count >> 9, true);
if (unlikely(ret))
goto err_put_bio;
if (unlikely(mapping->nrpages)) {
ret = write_invalidate_inode_pages_range(mapping,
req->ki_pos,
@ -2394,7 +2492,6 @@ err:
return ret;
err_put_bio:
bch2_pagecache_block_put(inode);
bch2_quota_reservation_put(c, inode, &dio->quota_res);
bio_put(bio);
inode_dio_end(&inode->v);
goto err;
@ -2446,19 +2543,21 @@ out:
* inode->ei_inode.bi_journal_seq won't be up to date since it's set in an
* insert trigger: look up the btree inode instead
*/
static int bch2_flush_inode(struct bch_fs *c, subvol_inum inum)
static int bch2_flush_inode(struct bch_fs *c,
struct bch_inode_info *inode)
{
struct bch_inode_unpacked inode;
struct bch_inode_unpacked u;
int ret;
if (c->opts.journal_flush_disabled)
return 0;
ret = bch2_inode_find_by_inum(c, inum, &inode);
ret = bch2_inode_find_by_inum(c, inode_inum(inode), &u);
if (ret)
return ret;
return bch2_journal_flush_seq(&c->journal, inode.bi_journal_seq);
return bch2_journal_flush_seq(&c->journal, u.bi_journal_seq) ?:
bch2_inode_flush_nocow_writes(c, inode);
}
int bch2_fsync(struct file *file, loff_t start, loff_t end, int datasync)
@ -2469,7 +2568,7 @@ int bch2_fsync(struct file *file, loff_t start, loff_t end, int datasync)
ret = file_write_and_wait_range(file, start, end);
ret2 = sync_inode_metadata(&inode->v, 1);
ret3 = bch2_flush_inode(c, inode_inum(inode));
ret3 = bch2_flush_inode(c, inode);
return bch2_err_class(ret ?: ret2 ?: ret3);
}
@ -2712,8 +2811,10 @@ int bch2_truncate(struct user_namespace *mnt_userns,
if (ret)
goto err;
WARN_ON(!test_bit(EI_INODE_ERROR, &inode->ei_flags) &&
inode->v.i_size < inode_u.bi_size);
WARN_ONCE(!test_bit(EI_INODE_ERROR, &inode->ei_flags) &&
inode->v.i_size < inode_u.bi_size,
"truncate spotted in mem i_size < btree i_size: %llu < %llu\n",
(u64) inode->v.i_size, inode_u.bi_size);
if (iattr->ia_size > inode->v.i_size) {
ret = bch2_extend(mnt_userns, inode, &inode_u, iattr);
@ -3015,7 +3116,7 @@ static int __bchfs_fallocate(struct bch_inode_info *inode, int mode,
struct btree_trans trans;
struct btree_iter iter;
struct bpos end_pos = POS(inode->v.i_ino, end_sector);
unsigned replicas = io_opts(c, &inode->ei_inode).data_replicas;
struct bch_io_opts opts = io_opts(c, &inode->ei_inode);
int ret = 0;
bch2_trans_init(&trans, c, BTREE_ITER_MAX, 512);
@ -3026,9 +3127,7 @@ static int __bchfs_fallocate(struct bch_inode_info *inode, int mode,
while (!ret && bkey_cmp(iter.pos, end_pos) < 0) {
s64 i_sectors_delta = 0;
struct disk_reservation disk_res = { 0 };
struct quota_res quota_res = { 0 };
struct bkey_i_reservation reservation;
struct bkey_s_c k;
unsigned sectors;
u32 snapshot;
@ -3047,8 +3146,8 @@ static int __bchfs_fallocate(struct bch_inode_info *inode, int mode,
goto bkey_err;
/* already reserved */
if (k.k->type == KEY_TYPE_reservation &&
bkey_s_c_to_reservation(k).v->nr_replicas >= replicas) {
if (bkey_extent_is_reservation(k) &&
bch2_bkey_nr_ptrs_fully_allocated(k) >= opts.data_replicas) {
bch2_btree_iter_advance(&iter);
continue;
}
@ -3059,16 +3158,12 @@ static int __bchfs_fallocate(struct bch_inode_info *inode, int mode,
continue;
}
bkey_reservation_init(&reservation.k_i);
reservation.k.type = KEY_TYPE_reservation;
reservation.k.p = k.k->p;
reservation.k.size = k.k->size;
/*
* XXX: for nocow mode, we should promote shared extents to
* unshared here
*/
bch2_cut_front(iter.pos, &reservation.k_i);
bch2_cut_back(end_pos, &reservation.k_i);
sectors = reservation.k.size;
reservation.v.nr_replicas = bch2_bkey_nr_ptrs_allocated(k);
sectors = bpos_min(k.k->p, end_pos).offset - iter.pos.offset;
if (!bkey_extent_is_allocation(k.k)) {
ret = bch2_quota_reservation_add(c, inode,
@ -3078,25 +3173,15 @@ static int __bchfs_fallocate(struct bch_inode_info *inode, int mode,
goto bkey_err;
}
if (reservation.v.nr_replicas < replicas ||
bch2_bkey_sectors_compressed(k)) {
ret = bch2_disk_reservation_get(c, &disk_res, sectors,
replicas, 0);
if (unlikely(ret))
goto bkey_err;
reservation.v.nr_replicas = disk_res.nr_replicas;
}
ret = bch2_extent_update(&trans, inode_inum(inode), &iter,
&reservation.k_i, &disk_res,
0, &i_sectors_delta, true);
ret = bch2_extent_fallocate(&trans, inode_inum(inode), &iter,
sectors, opts, &i_sectors_delta,
writepoint_hashed((unsigned long) current));
if (ret)
goto bkey_err;
i_sectors_acct(c, inode, &quota_res, i_sectors_delta);
bkey_err:
bch2_quota_reservation_put(c, inode, &quota_res);
bch2_disk_reservation_put(c, &disk_res);
if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
ret = 0;
}
@ -3337,7 +3422,7 @@ loff_t bch2_remap_file_range(struct file *file_src, loff_t pos_src,
if ((file_dst->f_flags & (__O_SYNC | O_DSYNC)) ||
IS_SYNC(file_inode(file_dst)))
ret = bch2_flush_inode(c, inode_inum(dst));
ret = bch2_flush_inode(c, dst);
err:
bch2_quota_reservation_put(c, dst, &quota_res);
bch2_unlock_inodes(INODE_LOCK|INODE_PAGECACHE_BLOCK, src, dst);
@ -3593,6 +3678,7 @@ loff_t bch2_llseek(struct file *file, loff_t offset, int whence)
void bch2_fs_fsio_exit(struct bch_fs *c)
{
bioset_exit(&c->nocow_flush_bioset);
bioset_exit(&c->dio_write_bioset);
bioset_exit(&c->dio_read_bioset);
bioset_exit(&c->writepage_bioset);
@ -3612,7 +3698,9 @@ int bch2_fs_fsio_init(struct bch_fs *c)
BIOSET_NEED_BVECS) ||
bioset_init(&c->dio_write_bioset,
4, offsetof(struct dio_write, op.wbio.bio),
BIOSET_NEED_BVECS))
BIOSET_NEED_BVECS) ||
bioset_init(&c->nocow_flush_bioset,
1, offsetof(struct nocow_flush, bio), 0))
ret = -ENOMEM;
pr_verbose_init(c->opts, "ret %i", ret);

3
libbcachefs/fs.c
View File

@ -810,6 +810,9 @@ static int bch2_fill_extent(struct bch_fs *c,
int flags2 = 0;
u64 offset = p.ptr.offset;
if (p.ptr.unwritten)
flags2 |= FIEMAP_EXTENT_UNWRITTEN;
if (p.crc.compression_type)
flags2 |= FIEMAP_EXTENT_ENCODED;
else

11
libbcachefs/fs.h
View File

@ -26,6 +26,17 @@ struct bch_inode_info {
u32 ei_subvol;
/*
* When we've been doing nocow writes we'll need to issue flushes to the
* underlying block devices
*
* XXX: a device may have had a flush issued by some other codepath. It
* would be better to keep for each device a sequence number that's
* incremented when we isusue a cache flush, and track here the sequence
* number that needs flushing.
*/
struct bch_devs_mask ei_devs_need_flush;
/* copy of inode in btree: */
struct bch_inode_unpacked ei_inode;
};

4
libbcachefs/fsck.c
View File

@ -1257,8 +1257,8 @@ static int check_extent(struct btree_trans *trans, struct btree_iter *iter,
continue;
if (fsck_err_on(!(i->inode.bi_flags & BCH_INODE_I_SIZE_DIRTY) &&
k.k->type != KEY_TYPE_reservation &&
k.k->p.offset > round_up(i->inode.bi_size, block_bytes(c)) >> 9, c,
k.k->p.offset > round_up(i->inode.bi_size, block_bytes(c)) >> 9 &&
!bkey_extent_is_reservation(k), c,
"extent type past end of inode %llu:%u, i_size %llu\n %s",
i->inode.bi_inum, i->snapshot, i->inode.bi_size,
(bch2_bkey_val_to_text(&buf, c, k), buf.buf))) {

9
libbcachefs/inode.c
View File

@ -733,7 +733,6 @@ static int bch2_inode_delete_keys(struct btree_trans *trans,
* iterator:
*/
bch2_trans_iter_init(trans, &iter, id, POS(inum.inum, 0),
BTREE_ITER_NOT_EXTENTS|
BTREE_ITER_INTENT);
while (1) {
@ -756,6 +755,14 @@ static int bch2_inode_delete_keys(struct btree_trans *trans,
bkey_init(&delete.k);
delete.k.p = iter.pos;
if (iter.flags & BTREE_ITER_IS_EXTENTS) {
bch2_key_resize(&delete.k, k.k->p.offset - iter.pos.offset);
ret = bch2_extent_trim_atomic(trans, &iter, &delete);
if (ret)
goto err;
}
ret = bch2_trans_update(trans, &iter, &delete, 0) ?:
bch2_trans_commit(trans, NULL, NULL,
BTREE_INSERT_NOFAIL);

2
libbcachefs/inode.h
View File

@ -157,6 +157,8 @@ io_opts(struct bch_fs *c, struct bch_inode_unpacked *inode)
struct bch_io_opts opts = bch2_opts_to_inode_opts(c->opts);
bch2_io_opts_apply(&opts, bch2_inode_opts_get(inode));
if (opts.nocow)
opts.compression = opts.background_compression = opts.data_checksum = opts.erasure_code;
return opts;
}

623
libbcachefs/io.c
View File

@ -233,62 +233,23 @@ int bch2_sum_sector_overwrites(struct btree_trans *trans,
return ret;
}
int bch2_extent_update(struct btree_trans *trans,
subvol_inum inum,
struct btree_iter *iter,
struct bkey_i *k,
struct disk_reservation *disk_res,
u64 new_i_size,
s64 *i_sectors_delta_total,
bool check_enospc)
static int bch2_extent_update_i_size_sectors(struct btree_trans *trans,
struct btree_iter *extent_iter,
u64 new_i_size,
s64 i_sectors_delta)
{
struct btree_iter inode_iter = { NULL };
struct btree_iter iter;
struct bkey_s_c inode_k;
struct bkey_s_c_inode_v3 inode;
struct bkey_i_inode_v3 *new_inode;
struct bpos next_pos;
bool usage_increasing;
s64 i_sectors_delta = 0, disk_sectors_delta = 0;
int ret;
/*
* This traverses us the iterator without changing iter->path->pos to
* search_key() (which is pos + 1 for extents): we want there to be a
* path already traversed at iter->pos because
* bch2_trans_extent_update() will use it to attempt extent merging
*/
ret = __bch2_btree_iter_traverse(iter);
if (ret)
return ret;
ret = bch2_extent_trim_atomic(trans, iter, k);
if (ret)
return ret;
new_i_size = min(k->k.p.offset << 9, new_i_size);
next_pos = k->k.p;
ret = bch2_sum_sector_overwrites(trans, iter, k,
&usage_increasing,
&i_sectors_delta,
&disk_sectors_delta);
if (ret)
return ret;
if (disk_res &&
disk_sectors_delta > (s64) disk_res->sectors) {
ret = bch2_disk_reservation_add(trans->c, disk_res,
disk_sectors_delta - disk_res->sectors,
!check_enospc || !usage_increasing
? BCH_DISK_RESERVATION_NOFAIL : 0);
if (ret)
return ret;
}
bch2_trans_iter_init(trans, &inode_iter, BTREE_ID_inodes,
SPOS(0, inum.inum, iter->snapshot),
bch2_trans_iter_init(trans, &iter, BTREE_ID_inodes,
SPOS(0,
extent_iter->pos.inode,
extent_iter->snapshot),
BTREE_ITER_INTENT|BTREE_ITER_CACHED);
inode_k = bch2_btree_iter_peek_slot(&inode_iter);
inode_k = bch2_btree_iter_peek_slot(&iter);
ret = bkey_err(inode_k);
if (unlikely(ret))
goto err;
@ -319,27 +280,205 @@ int bch2_extent_update(struct btree_trans *trans,
le64_add_cpu(&new_inode->v.bi_sectors, i_sectors_delta);
new_inode->k.p.snapshot = iter->snapshot;
new_inode->k.p.snapshot = iter.snapshot;
ret = bch2_trans_update(trans, &iter, &new_inode->k_i,
BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE);
err:
bch2_trans_iter_exit(trans, &iter);
return ret;
}
int bch2_extent_update(struct btree_trans *trans,
subvol_inum inum,
struct btree_iter *iter,
struct bkey_i *k,
struct disk_reservation *disk_res,
u64 new_i_size,
s64 *i_sectors_delta_total,
bool check_enospc)
{
struct bpos next_pos;
bool usage_increasing;
s64 i_sectors_delta = 0, disk_sectors_delta = 0;
int ret;
/*
* This traverses us the iterator without changing iter->path->pos to
* search_key() (which is pos + 1 for extents): we want there to be a
* path already traversed at iter->pos because
* bch2_trans_extent_update() will use it to attempt extent merging
*/
ret = __bch2_btree_iter_traverse(iter);
if (ret)
return ret;
ret = bch2_extent_trim_atomic(trans, iter, k);
if (ret)
return ret;
next_pos = k->k.p;
ret = bch2_sum_sector_overwrites(trans, iter, k,
&usage_increasing,
&i_sectors_delta,
&disk_sectors_delta);
if (ret)
return ret;
if (disk_res &&
disk_sectors_delta > (s64) disk_res->sectors) {
ret = bch2_disk_reservation_add(trans->c, disk_res,
disk_sectors_delta - disk_res->sectors,
!check_enospc || !usage_increasing
? BCH_DISK_RESERVATION_NOFAIL : 0);
if (ret)
return ret;
}
/*
* Note:
* We always have to do an inode updated - even when i_size/i_sectors
* We always have to do an inode update - even when i_size/i_sectors
* aren't changing - for fsync to work properly; fsync relies on
* inode->bi_journal_seq which is updated by the trigger code:
*/
ret = bch2_trans_update(trans, &inode_iter, &new_inode->k_i, 0) ?:
ret = bch2_extent_update_i_size_sectors(trans, iter,
min(k->k.p.offset << 9, new_i_size),
i_sectors_delta) ?:
bch2_trans_update(trans, iter, k, 0) ?:
bch2_trans_commit(trans, disk_res, NULL,
BTREE_INSERT_NOCHECK_RW|
BTREE_INSERT_NOFAIL);
if (unlikely(ret))
goto err;
return ret;
if (i_sectors_delta_total)
*i_sectors_delta_total += i_sectors_delta;
bch2_btree_iter_set_pos(iter, next_pos);
err:
bch2_trans_iter_exit(trans, &inode_iter);
return 0;
}
/* Overwrites whatever was present with zeroes: */
int bch2_extent_fallocate(struct btree_trans *trans,
subvol_inum inum,
struct btree_iter *iter,
unsigned sectors,
struct bch_io_opts opts,
s64 *i_sectors_delta,
struct write_point_specifier write_point)
{
struct bch_fs *c = trans->c;
struct disk_reservation disk_res = { 0 };
struct closure cl;
struct open_buckets open_buckets;
struct bkey_s_c k;
struct bkey_buf old, new;
bool have_reservation = false;
bool unwritten = opts.nocow &&
c->sb.version >= bcachefs_metadata_version_unwritten_extents;
int ret;
bch2_bkey_buf_init(&old);
bch2_bkey_buf_init(&new);
closure_init_stack(&cl);
open_buckets.nr = 0;
retry:
k = bch2_btree_iter_peek_slot(iter);
ret = bkey_err(k);
if (ret)
return ret;
sectors = min_t(u64, sectors, k.k->p.offset - iter->pos.offset);
if (!have_reservation) {
unsigned new_replicas =
max(0, (int) opts.data_replicas -
(int) bch2_bkey_nr_ptrs_fully_allocated(k));
/*
* Get a disk reservation before (in the nocow case) calling
* into the allocator:
*/
ret = bch2_disk_reservation_get(c, &disk_res, sectors, new_replicas, 0);
if (unlikely(ret))
goto out;
bch2_bkey_buf_reassemble(&old, c, k);
}
if (have_reservation) {
if (!bch2_extents_match(k, bkey_i_to_s_c(old.k)))
goto out;
bch2_key_resize(&new.k->k, sectors);
} else if (!unwritten) {
struct bkey_i_reservation *reservation;
bch2_bkey_buf_realloc(&new, c, sizeof(*reservation) / sizeof(u64));
reservation = bkey_reservation_init(new.k);
reservation->k.p = iter->pos;
bch2_key_resize(&reservation->k, sectors);
reservation->v.nr_replicas = opts.data_replicas;
} else {
struct bkey_i_extent *e;
struct bch_devs_list devs_have;
struct write_point *wp;
struct bch_extent_ptr *ptr;
devs_have.nr = 0;
bch2_bkey_buf_realloc(&new, c, BKEY_EXTENT_U64s_MAX);
e = bkey_extent_init(new.k);
e->k.p = iter->pos;
ret = bch2_alloc_sectors_start_trans(trans,
opts.foreground_target,
false,
write_point,
&devs_have,
opts.data_replicas,
opts.data_replicas,
RESERVE_none, 0, &cl, &wp);
if (ret == -EAGAIN) {
bch2_trans_unlock(trans);
closure_sync(&cl);
goto retry;
}
if (ret)
return ret;
sectors = min(sectors, wp->sectors_free);
bch2_key_resize(&e->k, sectors);
bch2_open_bucket_get(c, wp, &open_buckets);
bch2_alloc_sectors_append_ptrs(c, wp, &e->k_i, sectors, false);
bch2_alloc_sectors_done(c, wp);
extent_for_each_ptr(extent_i_to_s(e), ptr)
ptr->unwritten = true;
}
have_reservation = true;
ret = bch2_extent_update(trans, inum, iter, new.k, &disk_res,
0, i_sectors_delta, true);
out:
if ((atomic_read(&cl.remaining) & CLOSURE_REMAINING_MASK) != 1) {
bch2_trans_unlock(trans);
closure_sync(&cl);
}
if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) {
bch2_trans_begin(trans);
goto retry;
}
bch2_open_buckets_put(c, &open_buckets);
bch2_disk_reservation_put(c, &disk_res);
bch2_bkey_buf_exit(&new, c);
bch2_bkey_buf_exit(&old, c);
return ret;
}
@ -485,7 +624,8 @@ static int bch2_write_index_default(struct bch_write_op *op)
void bch2_submit_wbio_replicas(struct bch_write_bio *wbio, struct bch_fs *c,
enum bch_data_type type,
const struct bkey_i *k)
const struct bkey_i *k,
bool nocow)
{
struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(bkey_i_to_s_c(k));
const struct bch_extent_ptr *ptr;
@ -519,9 +659,11 @@ void bch2_submit_wbio_replicas(struct bch_write_bio *wbio, struct bch_fs *c,
n->c = c;
n->dev = ptr->dev;
n->have_ioref = bch2_dev_get_ioref(ca,
n->have_ioref = nocow || bch2_dev_get_ioref(ca,
type == BCH_DATA_btree ? READ : WRITE);
n->nocow = nocow;
n->submit_time = local_clock();
n->inode_offset = bkey_start_offset(&k->k);
n->bio.bi_iter.bi_sector = ptr->offset;
if (likely(n->have_ioref)) {
@ -624,8 +766,12 @@ static void __bch2_write_index(struct bch_write_op *op)
op->written += sectors_start - keylist_sectors(keys);
if (ret) {
bch_err_inum_ratelimited(c, op->pos.inode,
"write error while doing btree update: %s", bch2_err_str(ret));
struct bkey_i *k = bch2_keylist_front(&op->insert_keys);
bch_err_inum_offset_ratelimited(c,
k->k.p.inode, k->k.p.offset << 9,
"write error while doing btree update: %s",
bch2_err_str(ret));
goto err;
}
}
@ -692,13 +838,16 @@ static void bch2_write_endio(struct bio *bio)
if (bch2_dev_inum_io_err_on(bio->bi_status, ca,
op->pos.inode,
op->pos.offset - bio_sectors(bio), /* XXX definitely wrong */
wbio->inode_offset << 9,
"data write error: %s",
bch2_blk_status_to_str(bio->bi_status))) {
set_bit(wbio->dev, op->failed.d);
op->flags |= BCH_WRITE_IO_ERROR;
}
if (wbio->nocow)
set_bit(wbio->dev, op->devs_need_flush->d);
if (wbio->have_ioref) {
bch2_latency_acct(ca, wbio->submit_time, WRITE);
percpu_ref_put(&ca->io_ref);
@ -1122,6 +1271,301 @@ err:
return ret;
}
static bool bch2_extent_is_writeable(struct bch_write_op *op,
struct bkey_s_c k)
{
struct bch_fs *c = op->c;
struct bkey_s_c_extent e;
struct extent_ptr_decoded p;
const union bch_extent_entry *entry;
unsigned replicas = 0;
if (k.k->type != KEY_TYPE_extent)
return false;
e = bkey_s_c_to_extent(k);
extent_for_each_ptr_decode(e, p, entry) {
if (p.crc.csum_type ||
crc_is_compressed(p.crc) ||
p.has_ec)
return false;
replicas += bch2_extent_ptr_durability(c, &p);
}
return replicas >= op->opts.data_replicas;
}
static inline void bch2_nocow_write_unlock(struct bch_write_op *op)
{
struct bch_fs *c = op->c;
const struct bch_extent_ptr *ptr;
struct bkey_i *k;
for_each_keylist_key(&op->insert_keys, k) {
struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(bkey_i_to_s_c(k));
bkey_for_each_ptr(ptrs, ptr)
bch2_bucket_nocow_unlock(&c->nocow_locks,
PTR_BUCKET_POS(c, ptr),
BUCKET_NOCOW_LOCK_UPDATE);
}
}
static int bch2_nocow_write_convert_one_unwritten(struct btree_trans *trans,
struct btree_iter *iter,
struct bkey_i *orig,
struct bkey_s_c k,
u64 new_i_size)
{
struct bkey_i *new;
struct bkey_ptrs ptrs;
struct bch_extent_ptr *ptr;
int ret;
if (!bch2_extents_match(bkey_i_to_s_c(orig), k)) {
/* trace this */
return 0;
}
new = bch2_trans_kmalloc(trans, bkey_bytes(k.k));
ret = PTR_ERR_OR_ZERO(new);
if (ret)
return ret;
bkey_reassemble(new, k);
bch2_cut_front(bkey_start_pos(&orig->k), new);
bch2_cut_back(orig->k.p, new);
ptrs = bch2_bkey_ptrs(bkey_i_to_s(new));
bkey_for_each_ptr(ptrs, ptr)
ptr->unwritten = 0;
/*
* Note that we're not calling bch2_subvol_get_snapshot() in this path -
* that was done when we kicked off the write, and here it's important
* that we update the extent that we wrote to - even if a snapshot has
* since been created. The write is still outstanding, so we're ok
* w.r.t. snapshot atomicity:
*/
return bch2_extent_update_i_size_sectors(trans, iter,
min(new->k.p.offset << 9, new_i_size), 0) ?:
bch2_trans_update(trans, iter, new,
BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE);
}
static void bch2_nocow_write_convert_unwritten(struct bch_write_op *op)
{
struct bch_fs *c = op->c;
struct btree_trans trans;
struct btree_iter iter;
struct bkey_i *orig;
struct bkey_s_c k;
int ret;
bch2_trans_init(&trans, c, 0, 0);
for_each_keylist_key(&op->insert_keys, orig) {
ret = for_each_btree_key_commit(&trans, iter, BTREE_ID_extents,
bkey_start_pos(&orig->k),
BTREE_ITER_INTENT, k,
NULL, NULL, BTREE_INSERT_NOFAIL, ({
if (bkey_cmp(bkey_start_pos(k.k), orig->k.p) >= 0)
break;
bch2_nocow_write_convert_one_unwritten(&trans, &iter, orig, k, op->new_i_size);
}));
if (ret) {
struct bkey_i *k = bch2_keylist_front(&op->insert_keys);
bch_err_inum_offset_ratelimited(c,
k->k.p.inode, k->k.p.offset << 9,
"write error while doing btree update: %s",
bch2_err_str(ret));
op->error = ret;
break;
}
}
bch2_trans_exit(&trans);
}
static void __bch2_nocow_write_done(struct bch_write_op *op)
{
bch2_nocow_write_unlock(op);
if (unlikely(op->flags & BCH_WRITE_IO_ERROR)) {
op->error = -EIO;
} else if (unlikely(op->flags & BCH_WRITE_CONVERT_UNWRITTEN))
bch2_nocow_write_convert_unwritten(op);
}
static void bch2_nocow_write_done(struct closure *cl)
{
struct bch_write_op *op = container_of(cl, struct bch_write_op, cl);
__bch2_nocow_write_done(op);
bch2_write_done(cl);
}
static void bch2_nocow_write(struct bch_write_op *op)
{
struct bch_fs *c = op->c;
struct btree_trans trans;
struct btree_iter iter;
struct bkey_s_c k;
struct bkey_ptrs_c ptrs;
const struct bch_extent_ptr *ptr, *ptr2;
u32 snapshot;
int ret;
if (op->flags & BCH_WRITE_MOVE)
return;
bch2_trans_init(&trans, c, 0, 0);
retry:
bch2_trans_begin(&trans);
ret = bch2_subvolume_get_snapshot(&trans, op->subvol, &snapshot);
if (unlikely(ret))
goto err;
bch2_trans_iter_init(&trans, &iter, BTREE_ID_extents,
SPOS(op->pos.inode, op->pos.offset, snapshot),
BTREE_ITER_SLOTS);
while (1) {
struct bio *bio = &op->wbio.bio;
k = bch2_btree_iter_peek_slot(&iter);
ret = bkey_err(k);
if (ret)
break;
/* fall back to normal cow write path? */
if (unlikely(k.k->p.snapshot != snapshot ||
!bch2_extent_is_writeable(op, k)))
break;
if (bch2_keylist_realloc(&op->insert_keys,
op->inline_keys,
ARRAY_SIZE(op->inline_keys),
k.k->u64s))
break;
/* Get iorefs before dropping btree locks: */
ptrs = bch2_bkey_ptrs_c(k);
bkey_for_each_ptr(ptrs, ptr)
if (unlikely(!bch2_dev_get_ioref(bch_dev_bkey_exists(c, ptr->dev), WRITE)))
goto err_get_ioref;
/* Unlock before taking nocow locks, doing IO: */
bkey_reassemble(op->insert_keys.top, k);
bch2_trans_unlock(&trans);
bch2_cut_front(op->pos, op->insert_keys.top);
bch2_cut_back(POS(op->pos.inode, op->pos.offset + bio_sectors(bio)), op->insert_keys.top);
ptrs = bch2_bkey_ptrs_c(bkey_i_to_s_c(op->insert_keys.top));
bkey_for_each_ptr(ptrs, ptr) {
bch2_bucket_nocow_lock(&c->nocow_locks,
PTR_BUCKET_POS(c, ptr),
BUCKET_NOCOW_LOCK_UPDATE);
if (unlikely(ptr_stale(bch_dev_bkey_exists(c, ptr->dev), ptr)))
goto err_bucket_stale;
if (ptr->unwritten)
op->flags |= BCH_WRITE_CONVERT_UNWRITTEN;
}
bio = &op->wbio.bio;
if (k.k->p.offset < op->pos.offset + bio_sectors(bio)) {
bio = bio_split(bio, k.k->p.offset - op->pos.offset,
GFP_KERNEL, &c->bio_write);
wbio_init(bio)->put_bio = true;
bio->bi_opf = op->wbio.bio.bi_opf;
} else {
op->flags |= BCH_WRITE_DONE;
}
op->pos.offset += bio_sectors(bio);
op->written += bio_sectors(bio);
bio->bi_end_io = bch2_write_endio;
bio->bi_private = &op->cl;
bio->bi_opf |= REQ_OP_WRITE;
closure_get(&op->cl);
bch2_submit_wbio_replicas(to_wbio(bio), c, BCH_DATA_user,
op->insert_keys.top, true);
bch2_keylist_push(&op->insert_keys);
if (op->flags & BCH_WRITE_DONE)
break;
bch2_btree_iter_advance(&iter);
}
out:
bch2_trans_iter_exit(&trans, &iter);
err:
if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
goto retry;
if (ret) {
bch_err_inum_offset_ratelimited(c,
op->pos.inode,
op->pos.offset << 9,
"%s: btree lookup error %s",
__func__, bch2_err_str(ret));
op->error = ret;
op->flags |= BCH_WRITE_DONE;
}
bch2_trans_exit(&trans);
/* fallback to cow write path? */
if (!(op->flags & BCH_WRITE_DONE)) {
closure_sync(&op->cl);
__bch2_nocow_write_done(op);
op->insert_keys.top = op->insert_keys.keys;
} else if (op->flags & BCH_WRITE_SYNC) {
closure_sync(&op->cl);
bch2_nocow_write_done(&op->cl);
} else {
/*
* XXX
* needs to run out of process context because ei_quota_lock is
* a mutex
*/
continue_at(&op->cl, bch2_nocow_write_done, index_update_wq(op));
}
return;
err_get_ioref:
bkey_for_each_ptr(ptrs, ptr2) {
if (ptr2 == ptr)
break;
percpu_ref_put(&bch_dev_bkey_exists(c, ptr2->dev)->io_ref);
}
/* Fall back to COW path: */
goto out;
err_bucket_stale:
bkey_for_each_ptr(ptrs, ptr2) {
bch2_bucket_nocow_unlock(&c->nocow_locks,
PTR_BUCKET_POS(c, ptr2),
BUCKET_NOCOW_LOCK_UPDATE);
if (ptr2 == ptr)
break;
}
bkey_for_each_ptr(ptrs, ptr2)
percpu_ref_put(&bch_dev_bkey_exists(c, ptr2->dev)->io_ref);
/* We can retry this: */
ret = BCH_ERR_transaction_restart;
goto out;
}
static void __bch2_write(struct bch_write_op *op)
{
struct bch_fs *c = op->c;
@ -1131,6 +1575,12 @@ static void __bch2_write(struct bch_write_op *op)
int ret;
nofs_flags = memalloc_nofs_save();
if (unlikely(op->opts.nocow)) {
bch2_nocow_write(op);
if (op->flags & BCH_WRITE_DONE)
goto out_nofs_restore;
}
again:
memset(&op->failed, 0, sizeof(op->failed));
op->btree_update_ready = false;
@ -1202,7 +1652,7 @@ again:
key_to_write_offset);
bch2_submit_wbio_replicas(to_wbio(bio), c, BCH_DATA_user,
key_to_write);
key_to_write, false);
} while (ret);
/*
@ -1227,7 +1677,7 @@ again:
continue_at(&op->cl, bch2_write_index, NULL);
}
out_nofs_restore:
memalloc_nofs_restore(nofs_flags);
}
@ -1307,8 +1757,10 @@ void bch2_write(struct closure *cl)
wbio_init(bio)->put_bio = false;
if (bio->bi_iter.bi_size & (c->opts.block_size - 1)) {
bch_err_inum_ratelimited(c, op->pos.inode,
"misaligned write");
bch_err_inum_offset_ratelimited(c,
op->pos.inode,
op->pos.offset << 9,
"misaligned write");
op->error = -EIO;
goto err;
}
@ -1374,6 +1826,9 @@ static inline bool should_promote(struct bch_fs *c, struct bkey_s_c k,
if (bch2_bkey_has_target(c, k, opts.promote_target))
return false;
if (bkey_extent_is_unwritten(k))
return false;
if (bch2_target_congested(c, opts.promote_target)) {
/* XXX trace this */
return false;
@ -1889,20 +2344,25 @@ csum_err:
goto out;
}
bch2_dev_inum_io_error(ca, rbio->read_pos.inode, (u64) rbio->bvec_iter.bi_sector,
bch_err_inum_offset_ratelimited(ca,
rbio->read_pos.inode,
rbio->read_pos.offset << 9,
"data checksum error: expected %0llx:%0llx got %0llx:%0llx (type %s)",
rbio->pick.crc.csum.hi, rbio->pick.crc.csum.lo,
csum.hi, csum.lo, bch2_csum_types[crc.csum_type]);
bch2_io_error(ca);
bch2_rbio_error(rbio, READ_RETRY_AVOID, BLK_STS_IOERR);
goto out;
decompression_err:
bch_err_inum_ratelimited(c, rbio->read_pos.inode,
"decompression error");
bch_err_inum_offset_ratelimited(c, rbio->read_pos.inode,
rbio->read_pos.offset << 9,
"decompression error");
bch2_rbio_error(rbio, READ_ERR, BLK_STS_IOERR);
goto out;
decrypt_err:
bch_err_inum_ratelimited(c, rbio->read_pos.inode,
"decrypt error");
bch_err_inum_offset_ratelimited(c, rbio->read_pos.inode,
rbio->read_pos.offset << 9,
"decrypt error");
bch2_rbio_error(rbio, READ_ERR, BLK_STS_IOERR);
goto out;
}
@ -1977,7 +2437,9 @@ int __bch2_read_indirect_extent(struct btree_trans *trans,
if (k.k->type != KEY_TYPE_reflink_v &&
k.k->type != KEY_TYPE_indirect_inline_data) {
bch_err_inum_ratelimited(trans->c, orig_k->k->k.p.inode,
bch_err_inum_offset_ratelimited(trans->c,
orig_k->k->k.p.inode,
orig_k->k->k.p.offset << 9,
"%llu len %u points to nonexistent indirect extent %llu",
orig_k->k->k.p.offset,
orig_k->k->k.size,
@ -2063,8 +2525,9 @@ retry_pick:
goto hole;
if (pick_ret < 0) {
bch_err_inum_ratelimited(c, k.k->p.inode,
"no device to read from");
bch_err_inum_offset_ratelimited(c,
read_pos.inode, read_pos.offset << 9,
"no device to read from");
goto err;
}
@ -2243,8 +2706,10 @@ get_bio:
if (!rbio->pick.idx) {
if (!rbio->have_ioref) {
bch_err_inum_ratelimited(c, k.k->p.inode,
"no device to read from");
bch_err_inum_offset_ratelimited(c,
read_pos.inode,
read_pos.offset << 9,
"no device to read from");
bch2_rbio_error(rbio, READ_RETRY_AVOID, BLK_STS_IOERR);
goto out;
}
@ -2412,8 +2877,9 @@ err:
bch2_bkey_buf_exit(&sk, c);
if (ret) {
bch_err_inum_ratelimited(c, inum.inum,
"read error %i from btree lookup", ret);
bch_err_inum_offset_ratelimited(c, inum.inum,
bvec_iter.bi_sector << 9,
"read error %i from btree lookup", ret);
rbio->bio.bi_status = BLK_STS_IOERR;
bch2_rbio_done(rbio);
}
@ -2431,6 +2897,11 @@ void bch2_fs_io_exit(struct bch_fs *c)
int bch2_fs_io_init(struct bch_fs *c)
{
unsigned i;
for (i = 0; i < ARRAY_SIZE(c->nocow_locks.l); i++)
two_state_lock_init(&c->nocow_locks.l[i]);
if (bioset_init(&c->bio_read, 1, offsetof(struct bch_read_bio, bio),
BIOSET_NEED_BVECS) ||
bioset_init(&c->bio_read_split, 1, offsetof(struct bch_read_bio, bio),

9
libbcachefs/io.h
View File

@ -18,7 +18,7 @@ void bch2_bio_alloc_pages_pool(struct bch_fs *, struct bio *, size_t);
void bch2_latency_acct(struct bch_dev *, u64, int);
void bch2_submit_wbio_replicas(struct bch_write_bio *, struct bch_fs *,
enum bch_data_type, const struct bkey_i *);
enum bch_data_type, const struct bkey_i *, bool);
#define BLK_STS_REMOVED ((__force blk_status_t)128)
@ -39,6 +39,7 @@ enum bch_write_flags {
/* Internal: */
BCH_WRITE_DONE = (1 << 10),
BCH_WRITE_IO_ERROR = (1 << 11),
BCH_WRITE_CONVERT_UNWRITTEN = (1 << 12),
};
static inline struct workqueue_struct *index_update_wq(struct bch_write_op *op)
@ -53,6 +54,9 @@ int bch2_sum_sector_overwrites(struct btree_trans *, struct btree_iter *,
int bch2_extent_update(struct btree_trans *, subvol_inum,
struct btree_iter *, struct bkey_i *,
struct disk_reservation *, u64, s64 *, bool);
int bch2_extent_fallocate(struct btree_trans *, subvol_inum, struct btree_iter *,
unsigned, struct bch_io_opts, s64 *,
struct write_point_specifier);
int bch2_fpunch_at(struct btree_trans *, struct btree_iter *,
subvol_inum, u64, s64 *);
@ -66,7 +70,7 @@ static inline void bch2_write_op_init(struct bch_write_op *op, struct bch_fs *c,
op->flags = 0;
op->written = 0;
op->error = 0;
op->csum_type = bch2_data_checksum_type(c, opts.data_checksum);
op->csum_type = bch2_data_checksum_type(c, opts);
op->compression_type = bch2_compression_opt_to_type[opts.compression];
op->nr_replicas = 0;
op->nr_replicas_required = c->opts.data_replicas_required;
@ -83,6 +87,7 @@ static inline void bch2_write_op_init(struct bch_write_op *op, struct bch_fs *c,
op->res = (struct disk_reservation) { 0 };
op->new_i_size = U64_MAX;
op->i_sectors_delta = 0;
op->devs_need_flush = NULL;
}
void bch2_write(struct closure *);

8
libbcachefs/io_types.h
View File

@ -87,6 +87,7 @@ struct bch_write_bio {
struct bch_write_bio *parent;
u64 submit_time;
u64 inode_offset;
struct bch_devs_list failed;
u8 dev;
@ -95,6 +96,7 @@ struct bch_write_bio {
bounce:1,
put_bio:1,
have_ioref:1,
nocow:1,
used_mempool:1,
first_btree_write:1;
@ -148,6 +150,12 @@ struct bch_write_op {
struct keylist insert_keys;
u64 inline_keys[BKEY_EXTENT_U64s_MAX * 2];
/*
* Bitmask of devices that have had nocow writes issued to them since
* last flush:
*/
struct bch_devs_mask *devs_need_flush;
/* Must be last: */
struct bch_write_bio wbio;
};

8
libbcachefs/journal_io.c
View File

@ -1269,10 +1269,10 @@ int bch2_journal_read(struct bch_fs *c, u64 *blacklist_seq, u64 *start_seq)
struct bch_dev *ca = bch_dev_bkey_exists(c, i->ptrs[ptr].dev);
if (!i->ptrs[ptr].csum_good)
printk(KERN_ERR "bcachefs (%s) sector %llu: invalid journal checksum, seq %llu%s\n",
ca->name, i->ptrs[ptr].sector,
le64_to_cpu(i->j.seq),
i->csum_good ? " (had good copy on another device)" : "");
bch_err_dev_offset(ca, i->ptrs[ptr].sector,
"invalid journal checksum, seq %llu%s",
le64_to_cpu(i->j.seq),
i->csum_good ? " (had good copy on another device)" : "");
}
ret = jset_validate(c,

66
libbcachefs/move.c
View File

@ -261,6 +261,12 @@ static int bch2_move_extent(struct btree_trans *trans,
if (!percpu_ref_tryget_live(&c->writes))
return -EROFS;
/*
* Before memory allocations & taking nocow locks in
* bch2_data_update_init():
*/
bch2_trans_unlock(trans);
/* write path might have to decompress data: */
bkey_for_each_ptr_decode(k.k, ptrs, p, entry)
sectors = max_t(unsigned, sectors, p.crc.uncompressed_size);
@ -296,7 +302,7 @@ static int bch2_move_extent(struct btree_trans *trans,
ret = bch2_data_update_init(c, &io->write, ctxt->wp, io_opts,
data_opts, btree_id, k);
if (ret)
if (ret && ret != -BCH_ERR_unwritten_extent_update)
goto err_free_pages;
io->write.ctxt = ctxt;
@ -304,6 +310,15 @@ static int bch2_move_extent(struct btree_trans *trans,
atomic64_inc(&ctxt->stats->keys_moved);
atomic64_add(k.k->size, &ctxt->stats->sectors_moved);
if (ret == -BCH_ERR_unwritten_extent_update) {
bch2_update_unwritten_extent(trans, &io->write);
move_free(io);
return 0;
}
BUG_ON(ret);
this_cpu_add(c->counters[BCH_COUNTER_io_move], k.k->size);
this_cpu_add(c->counters[BCH_COUNTER_move_extent_read], k.k->size);
trace_move_extent_read(k.k);
@ -499,6 +514,7 @@ static int __bch2_move_data(struct moving_context *ctxt,
*/
bch2_bkey_buf_reassemble(&sk, c, k);
k = bkey_i_to_s_c(sk.k);
bch2_trans_unlock(&trans);
ret2 = bch2_move_extent(&trans, &iter, ctxt, io_opts,
btree_id, k, data_opts);
@ -573,6 +589,9 @@ static int verify_bucket_evacuated(struct btree_trans *trans, struct bpos bucket
struct bch_fs *c = trans->c;
struct btree_iter iter;
struct bkey_s_c k;
struct printbuf buf = PRINTBUF;
struct bch_backpointer bp;
u64 bp_offset = 0;
int ret;
bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc,
@ -586,24 +605,53 @@ again:
if (a.v->gen == gen &&
a.v->dirty_sectors) {
struct printbuf buf = PRINTBUF;
if (a.v->data_type == BCH_DATA_btree) {
bch2_trans_unlock(trans);
if (bch2_btree_interior_updates_flush(c))
goto again;
goto failed_to_evacuate;
}
prt_str(&buf, "failed to evacuate bucket ");
bch2_bkey_val_to_text(&buf, c, k);
bch_err(c, "%s", buf.buf);
printbuf_exit(&buf);
}
}
bch2_trans_iter_exit(trans, &iter);
return ret;
failed_to_evacuate:
bch2_trans_iter_exit(trans, &iter);
prt_printf(&buf, bch2_log_msg(c, "failed to evacuate bucket "));
bch2_bkey_val_to_text(&buf, c, k);
while (1) {
bch2_trans_begin(trans);
ret = bch2_get_next_backpointer(trans, bucket, gen,
&bp_offset, &bp,
BTREE_ITER_CACHED);
if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
continue;
if (ret)
break;
if (bp_offset == U64_MAX)
break;
k = bch2_backpointer_get_key(trans, &iter,
bucket, bp_offset, bp);
ret = bkey_err(k);
if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
continue;
if (ret)
break;
if (!k.k)
continue;
prt_newline(&buf);
bch2_bkey_val_to_text(&buf, c, k);
bch2_trans_iter_exit(trans, &iter);
}
bch2_print_string_as_lines(KERN_ERR, buf.buf);
printbuf_exit(&buf);
return 0;
}
int __bch2_evacuate_bucket(struct moving_context *ctxt,

16
libbcachefs/nocow_locking.c Normal file
View File

@ -0,0 +1,16 @@
// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "nocow_locking.h"
#include "util.h"
void __bch2_bucket_nocow_lock(struct bucket_nocow_lock_table *t,
struct bpos bucket, int flags)
{
struct bch_fs *c = container_of(t, struct bch_fs, nocow_locks);
two_state_lock_t *l = bucket_nocow_lock(t, bucket);
u64 start_time = local_clock();
bch2_two_state_lock(l, flags & BUCKET_NOCOW_LOCK_UPDATE);
bch2_time_stats_update(&c->times[BCH_TIME_nocow_lock_contended], start_time);
}

55
libbcachefs/nocow_locking.h Normal file
View File

@ -0,0 +1,55 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_NOCOW_LOCKING_H
#define _BCACHEFS_NOCOW_LOCKING_H
#include "bcachefs_format.h"
#include "two_state_shared_lock.h"
#include <linux/siphash.h>
#define BUCKET_NOCOW_LOCKS (1U << 10)
struct bucket_nocow_lock_table {
siphash_key_t key;
two_state_lock_t l[BUCKET_NOCOW_LOCKS];
};
#define BUCKET_NOCOW_LOCK_UPDATE (1 << 0)
static inline two_state_lock_t *bucket_nocow_lock(struct bucket_nocow_lock_table *t,
struct bpos bucket)
{
u64 dev_bucket = bucket.inode << 56 | bucket.offset;
unsigned h = siphash_1u64(dev_bucket, &t->key);
return t->l + (h & (BUCKET_NOCOW_LOCKS - 1));
}
static inline bool bch2_bucket_nocow_is_locked(struct bucket_nocow_lock_table *t,
struct bpos bucket)
{
two_state_lock_t *l = bucket_nocow_lock(t, bucket);
return atomic_long_read(&l->v) != 0;
}
static inline void bch2_bucket_nocow_unlock(struct bucket_nocow_lock_table *t,
struct bpos bucket, int flags)
{
two_state_lock_t *l = bucket_nocow_lock(t, bucket);
bch2_two_state_unlock(l, flags & BUCKET_NOCOW_LOCK_UPDATE);
}
void __bch2_bucket_nocow_lock(struct bucket_nocow_lock_table *, struct bpos, int);
static inline void bch2_bucket_nocow_lock(struct bucket_nocow_lock_table *t,
struct bpos bucket, int flags)
{
two_state_lock_t *l = bucket_nocow_lock(t, bucket);
if (!bch2_two_state_trylock(l, flags & BUCKET_NOCOW_LOCK_UPDATE))
__bch2_bucket_nocow_lock(t, bucket, flags);
}
#endif /* _BCACHEFS_NOCOW_LOCKING_H */

7
libbcachefs/opts.h
View File

@ -382,6 +382,13 @@ enum opt_type {
OPT_BOOL(), \
BCH2_NO_SB_OPT, false, \
NULL, NULL) \
x(nocow, u8, \
OPT_FS|OPT_FORMAT|OPT_MOUNT|OPT_RUNTIME|OPT_INODE, \
OPT_BOOL(), \
BCH_SB_NOCOW, false, \
NULL, "Nocow mode: Writes will be done in place when possible.\n"\
"Snapshots and reflink will still caused writes to be COW\n"\
"Implicitly disables data checksumming, compression and encryption")\
x(fs_size, u64, \
OPT_DEVICE, \
OPT_UINT(0, S64_MAX), \

27
libbcachefs/quota.c
View File

@ -364,16 +364,16 @@ int bch2_quota_acct(struct bch_fs *c, struct bch_qid qid,
memset(&msgs, 0, sizeof(msgs));
for_each_set_qtype(c, i, q, qtypes) {
mq[i] = genradix_ptr_alloc(&q->table, qid.q[i], GFP_KERNEL);
if (!mq[i])
return -ENOMEM;
}
for_each_set_qtype(c, i, q, qtypes)
mutex_lock_nested(&q->lock, i);
for_each_set_qtype(c, i, q, qtypes) {
mq[i] = genradix_ptr_alloc(&q->table, qid.q[i], GFP_NOFS);
if (!mq[i]) {
ret = -ENOMEM;
goto err;
}
ret = bch2_quota_check_limit(c, i, mq[i], &msgs, counter, v, mode);
if (ret)
goto err;
@ -416,18 +416,17 @@ int bch2_quota_transfer(struct bch_fs *c, unsigned qtypes,
memset(&msgs, 0, sizeof(msgs));
for_each_set_qtype(c, i, q, qtypes) {
src_q[i] = genradix_ptr_alloc(&q->table, src.q[i], GFP_KERNEL);
dst_q[i] = genradix_ptr_alloc(&q->table, dst.q[i], GFP_KERNEL);
if (!src_q[i] || !dst_q[i])
return -ENOMEM;
}
for_each_set_qtype(c, i, q, qtypes)
mutex_lock_nested(&q->lock, i);
for_each_set_qtype(c, i, q, qtypes) {
src_q[i] = genradix_ptr_alloc(&q->table, src.q[i], GFP_NOFS);
dst_q[i] = genradix_ptr_alloc(&q->table, dst.q[i], GFP_NOFS);
if (!src_q[i] || !dst_q[i]) {
ret = -ENOMEM;
goto err;
}
ret = bch2_quota_check_limit(c, i, dst_q[i], &msgs, Q_SPC,
dst_q[i]->c[Q_SPC].v + space,
mode);

3
libbcachefs/reflink.c
View File

@ -255,6 +255,9 @@ static struct bkey_s_c get_next_src(struct btree_iter *iter, struct bpos end)
if (bkey_cmp(iter->pos, end) >= 0)
break;
if (bkey_extent_is_unwritten(k))
continue;
if (bkey_extent_is_data(k.k))
return k;
}

7
libbcachefs/super.h
View File

@ -88,9 +88,10 @@ static inline void bch2_dev_list_drop_dev(struct bch_devs_list *devs,
static inline void bch2_dev_list_add_dev(struct bch_devs_list *devs,
unsigned dev)
{
BUG_ON(bch2_dev_list_has_dev(*devs, dev));
BUG_ON(devs->nr >= ARRAY_SIZE(devs->devs));
devs->devs[devs->nr++] = dev;
if (!bch2_dev_list_has_dev(*devs, dev)) {
BUG_ON(devs->nr >= ARRAY_SIZE(devs->devs));
devs->devs[devs->nr++] = dev;
}
}
static inline struct bch_devs_list bch2_dev_list_single(unsigned dev)

26
libbcachefs/util.h
View File

@ -23,33 +23,9 @@
struct closure;
#ifdef CONFIG_BCACHEFS_DEBUG
#define EBUG_ON(cond) BUG_ON(cond)
#define atomic_dec_bug(v) BUG_ON(atomic_dec_return(v) < 0)
#define atomic_inc_bug(v, i) BUG_ON(atomic_inc_return(v) <= i)
#define atomic_sub_bug(i, v) BUG_ON(atomic_sub_return(i, v) < 0)
#define atomic_add_bug(i, v) BUG_ON(atomic_add_return(i, v) < 0)
#define atomic_long_dec_bug(v) BUG_ON(atomic_long_dec_return(v) < 0)
#define atomic_long_sub_bug(i, v) BUG_ON(atomic_long_sub_return(i, v) < 0)
#define atomic64_dec_bug(v) BUG_ON(atomic64_dec_return(v) < 0)
#define atomic64_inc_bug(v, i) BUG_ON(atomic64_inc_return(v) <= i)
#define atomic64_sub_bug(i, v) BUG_ON(atomic64_sub_return(i, v) < 0)
#define atomic64_add_bug(i, v) BUG_ON(atomic64_add_return(i, v) < 0)
#else /* DEBUG */
#else
#define EBUG_ON(cond)
#define atomic_dec_bug(v) atomic_dec(v)
#define atomic_inc_bug(v, i) atomic_inc(v)
#define atomic_sub_bug(i, v) atomic_sub(i, v)
#define atomic_add_bug(i, v) atomic_add(i, v)
#define atomic_long_dec_bug(v) atomic_long_dec(v)
#define atomic_long_sub_bug(i, v) atomic_long_sub(i, v)
#define atomic64_dec_bug(v) atomic64_dec(v)
#define atomic64_inc_bug(v, i) atomic64_inc(v)
#define atomic64_sub_bug(i, v) atomic64_sub(i, v)
#define atomic64_add_bug(i, v) atomic64_add(i, v)
#endif
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__