diff --git a/.bcachefs_revision b/.bcachefs_revision index 7b6e85f0..b0cfa36c 100644 --- a/.bcachefs_revision +++ b/.bcachefs_revision @@ -1 +1 @@ -0c67e96b72019e02c92d2aed58de4c8b98e07d16 +115e433659fea5a1fea1dec5a3de77312cf2da8c diff --git a/libbcachefs/Makefile b/libbcachefs/Makefile index 1e87eee9..879f3b59 100644 --- a/libbcachefs/Makefile +++ b/libbcachefs/Makefile @@ -99,9 +99,12 @@ bcachefs-y := \ util.o \ varint.o \ xattr.o \ - vendor/closure.o + vendor/closure.o \ + vendor/min_heap.o +ifndef CONFIG_BCACHEFS_DKMS obj-$(CONFIG_MEAN_AND_VARIANCE_UNIT_TEST) += mean_and_variance_test.o +endif # Silence "note: xyz changed in GCC X.X" messages subdir-ccflags-y += $(call cc-disable-warning, psabi) diff --git a/libbcachefs/acl.c b/libbcachefs/acl.c index 3befa1f3..a1df0ec2 100644 --- a/libbcachefs/acl.c +++ b/libbcachefs/acl.c @@ -57,7 +57,7 @@ void bch2_acl_to_text(struct printbuf *out, const void *value, size_t size) } } -#ifdef CONFIG_BCACHEFS_POSIX_ACL +#ifndef NO_BCACHEFS_FS #include "fs.h" @@ -437,4 +437,4 @@ err: return ret; } -#endif /* CONFIG_BCACHEFS_POSIX_ACL */ +#endif /* NO_BCACHEFS_FS */ diff --git a/libbcachefs/acl.h b/libbcachefs/acl.h index fe730a6b..01cf21c5 100644 --- a/libbcachefs/acl.h +++ b/libbcachefs/acl.h @@ -26,7 +26,7 @@ typedef struct { void bch2_acl_to_text(struct printbuf *, const void *, size_t); -#ifdef CONFIG_BCACHEFS_POSIX_ACL +#ifndef NO_BCACHEFS_FS struct posix_acl *bch2_get_acl(struct inode *, int, bool); @@ -55,6 +55,6 @@ static inline int bch2_acl_chmod(struct btree_trans *trans, subvol_inum inum, return 0; } -#endif /* CONFIG_BCACHEFS_POSIX_ACL */ +#endif /* NO_BCACHEFS_FS */ #endif /* _BCACHEFS_ACL_H */ diff --git a/libbcachefs/btree_journal_iter.c b/libbcachefs/btree_journal_iter.c index a6f344fa..73fa8513 100644 --- a/libbcachefs/btree_journal_iter.c +++ b/libbcachefs/btree_journal_iter.c @@ -838,8 +838,10 @@ void bch2_shoot_down_journal_keys(struct bch_fs *c, enum btree_id btree, bpos_ge(k->k.p, start) && bpos_le(k->k.p, end))) keys->data[dst++] = *i; - else if (i->allocated) + else if (i->allocated) { kfree(i->allocated_k); + i->allocated_k = NULL; + } } keys->nr = keys->gap = dst; } diff --git a/libbcachefs/btree_node_scan.c b/libbcachefs/btree_node_scan.c index c0dff992..433d4985 100644 --- a/libbcachefs/btree_node_scan.c +++ b/libbcachefs/btree_node_scan.c @@ -12,7 +12,6 @@ #include "recovery_passes.h" #include -#include #include #include diff --git a/libbcachefs/clock_types.h b/libbcachefs/clock_types.h index 37554e45..d12c11dc 100644 --- a/libbcachefs/clock_types.h +++ b/libbcachefs/clock_types.h @@ -2,7 +2,7 @@ #ifndef _BCACHEFS_CLOCK_TYPES_H #define _BCACHEFS_CLOCK_TYPES_H -#include "util.h" +#include "vendor/min_heap.h" #define NR_IO_TIMERS (BCH_SB_MEMBERS_MAX * 3) diff --git a/libbcachefs/data_update.c b/libbcachefs/data_update.c index 155c1ad4..62d5d17d 100644 --- a/libbcachefs/data_update.c +++ b/libbcachefs/data_update.c @@ -91,8 +91,10 @@ bool __bkey_nocow_lock(struct bch_fs *c, struct moving_context *ctxt, struct bke move_ctxt_wait_event(ctxt, (locked = bch2_bucket_nocow_trylock(&c->nocow_locks, bucket, 0)) || list_empty(&ctxt->ios)); - if (!locked) + if (!locked) { + bch2_trans_unlock(ctxt->trans); bch2_bucket_nocow_lock(&c->nocow_locks, bucket, 0); + } } return true; } diff --git a/libbcachefs/errcode.h b/libbcachefs/errcode.h index c4344a1d..cbf1eedd 100644 --- a/libbcachefs/errcode.h +++ b/libbcachefs/errcode.h @@ -188,6 +188,11 @@ x(BCH_ERR_recovery_will_run, recovery_pass_will_run) \ x(0, data_update_done) \ x(0, bkey_was_deleted) \ + x(0, bucket_not_moveable) \ + x(BCH_ERR_bucket_not_moveable, bucket_not_moveable_dev_not_rw) \ + x(BCH_ERR_bucket_not_moveable, bucket_not_moveable_bucket_open) \ + x(BCH_ERR_bucket_not_moveable, bucket_not_moveable_bp_mismatch) \ + x(BCH_ERR_bucket_not_moveable, bucket_not_moveable_lru_race) \ x(BCH_ERR_data_update_done, data_update_done_would_block) \ x(BCH_ERR_data_update_done, data_update_done_unwritten) \ x(BCH_ERR_data_update_done, data_update_done_no_writes_needed) \ diff --git a/libbcachefs/fs.c b/libbcachefs/fs.c index f1849eb8..414adf86 100644 --- a/libbcachefs/fs.c +++ b/libbcachefs/fs.c @@ -541,11 +541,9 @@ __bch2_create(struct mnt_idmap *idmap, * preallocate acls + vfs inode before btree transaction, so that * nothing can fail after the transaction succeeds: */ -#ifdef CONFIG_BCACHEFS_POSIX_ACL ret = posix_acl_create(&dir->v, &mode, &default_acl, &acl); if (ret) return ERR_PTR(ret); -#endif inode = __bch2_new_inode(c, GFP_NOFS); if (unlikely(!inode)) { @@ -1751,10 +1749,8 @@ static const struct inode_operations bch_file_inode_operations = { .setattr = bch2_setattr, .fiemap = bch2_fiemap, .listxattr = bch2_xattr_list, -#ifdef CONFIG_BCACHEFS_POSIX_ACL .get_inode_acl = bch2_get_acl, .set_acl = bch2_set_acl, -#endif .fileattr_get = bch2_fileattr_get, .fileattr_set = bch2_fileattr_set, }; @@ -1773,10 +1769,8 @@ static const struct inode_operations bch_dir_inode_operations = { .setattr = bch2_setattr, .tmpfile = bch2_tmpfile, .listxattr = bch2_xattr_list, -#ifdef CONFIG_BCACHEFS_POSIX_ACL .get_inode_acl = bch2_get_acl, .set_acl = bch2_set_acl, -#endif .fileattr_get = bch2_fileattr_get, .fileattr_set = bch2_fileattr_set, }; @@ -1797,10 +1791,8 @@ static const struct inode_operations bch_symlink_inode_operations = { .getattr = bch2_getattr, .setattr = bch2_setattr, .listxattr = bch2_xattr_list, -#ifdef CONFIG_BCACHEFS_POSIX_ACL .get_inode_acl = bch2_get_acl, .set_acl = bch2_set_acl, -#endif .fileattr_get = bch2_fileattr_get, .fileattr_set = bch2_fileattr_set, }; @@ -1809,10 +1801,8 @@ static const struct inode_operations bch_special_inode_operations = { .getattr = bch2_getattr, .setattr = bch2_setattr, .listxattr = bch2_xattr_list, -#ifdef CONFIG_BCACHEFS_POSIX_ACL .get_inode_acl = bch2_get_acl, .set_acl = bch2_set_acl, -#endif .fileattr_get = bch2_fileattr_get, .fileattr_set = bch2_fileattr_set, }; @@ -2546,10 +2536,8 @@ got_sb: c->dev = sb->s_dev; -#ifdef CONFIG_BCACHEFS_POSIX_ACL if (c->opts.acl) sb->s_flags |= SB_POSIXACL; -#endif sb->s_shrink->seeks = 0; diff --git a/libbcachefs/movinggc.c b/libbcachefs/movinggc.c index f36d60b8..0f7e3568 100644 --- a/libbcachefs/movinggc.c +++ b/libbcachefs/movinggc.c @@ -62,33 +62,51 @@ static int bch2_bucket_is_movable(struct btree_trans *trans, { struct bch_fs *c = trans->c; - if (bch2_bucket_is_open(c, b->k.bucket.inode, b->k.bucket.offset)) + /* + * Valid bucket? + * + * XXX: we should kill the LRU entry here if it's not + */ + CLASS(bch2_dev_bucket_tryget, ca)(c, b->k.bucket); + if (!ca) return 0; + if (ca->mi.state != BCH_MEMBER_STATE_rw || + !bch2_dev_is_online(ca)) { + bch_err_throw(c, bucket_not_moveable_dev_not_rw); + return 0; + } + + /* Bucket still being written? */ + if (bch2_bucket_is_open(c, b->k.bucket.inode, b->k.bucket.offset)) { + bch_err_throw(c, bucket_not_moveable_bucket_open); + return 0; + } + + /* We won't be able to evacuate it if there's missing backpointers */ + if (bch2_bucket_bitmap_test(&ca->bucket_backpointer_mismatch, b->k.bucket.offset)) { + bch_err_throw(c, bucket_not_moveable_bp_mismatch); + return 0; + } + CLASS(btree_iter, iter)(trans, BTREE_ID_alloc, b->k.bucket, BTREE_ITER_cached); struct bkey_s_c k = bch2_btree_iter_peek_slot(&iter); int ret = bkey_err(k); if (ret) return ret; - CLASS(bch2_dev_bucket_tryget, ca)(c, k.k->p); - if (!ca) - return 0; - - if (bch2_bucket_bitmap_test(&ca->bucket_backpointer_mismatch, b->k.bucket.offset)) - return 0; - - if (ca->mi.state != BCH_MEMBER_STATE_rw || - !bch2_dev_is_online(ca)) - return 0; - struct bch_alloc_v4 _a; const struct bch_alloc_v4 *a = bch2_alloc_to_v4(k, &_a); b->k.gen = a->gen; b->sectors = bch2_bucket_sectors_dirty(*a); u64 lru_idx = alloc_lru_idx_fragmentation(*a, ca); - return lru_idx && lru_idx <= time; + if (!lru_idx || lru_idx > time) { + bch_err_throw(c, bucket_not_moveable_lru_race); + return 0; + } + + return true; } static void move_bucket_free(struct buckets_in_flight *list, diff --git a/libbcachefs/namei.c b/libbcachefs/namei.c index 5c321a0d..6d31508c 100644 --- a/libbcachefs/namei.c +++ b/libbcachefs/namei.c @@ -980,7 +980,7 @@ int bch2_check_inode_has_case_insensitive(struct btree_trans *trans, ret = bch2_inum_snapshot_to_path(trans, inode->bi_inum, inode->bi_snapshot, snapshot_overwrites, &buf); if (ret) - return ret; + goto out; if (fsck_err(trans, inode_has_case_insensitive_not_set, "%s", buf.buf)) { inode->bi_flags |= BCH_INODE_has_case_insensitive; @@ -999,14 +999,14 @@ int bch2_check_inode_has_case_insensitive(struct btree_trans *trans, if (dir.bi_parent_subvol) { ret = bch2_subvolume_get_snapshot(trans, dir.bi_parent_subvol, &snapshot); if (ret) - return ret; + goto out; snapshot_overwrites = NULL; } ret = bch2_inode_find_by_inum_snapshot(trans, dir.bi_dir, snapshot, &dir, 0); if (ret) - return ret; + goto out; if (!(dir.bi_flags & BCH_INODE_has_case_insensitive)) { prt_printf(&buf, "parent of casefolded dir with has_case_insensitive not set\n"); @@ -1014,13 +1014,13 @@ int bch2_check_inode_has_case_insensitive(struct btree_trans *trans, ret = bch2_inum_snapshot_to_path(trans, dir.bi_inum, dir.bi_snapshot, snapshot_overwrites, &buf); if (ret) - return ret; + goto out; if (fsck_err(trans, inode_parent_has_case_insensitive_not_set, "%s", buf.buf)) { dir.bi_flags |= BCH_INODE_has_case_insensitive; ret = __bch2_fsck_write_inode(trans, &dir); if (ret) - return ret; + goto out; } } @@ -1033,13 +1033,13 @@ int bch2_check_inode_has_case_insensitive(struct btree_trans *trans, } out: fsck_err: + bch_err_fn(trans->c, ret); if (ret) return ret; - if (repairing_parents) { + if (repairing_parents) return bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc) ?: bch_err_throw(trans->c, transaction_restart_nested); - } return 0; } diff --git a/libbcachefs/opts.c b/libbcachefs/opts.c index bd5faafc..b9bac9fb 100644 --- a/libbcachefs/opts.c +++ b/libbcachefs/opts.c @@ -649,10 +649,6 @@ int bch2_parse_one_mount_opt(struct bch_fs *c, struct bch_opts *opts, if (!(bch2_opt_table[id].flags & OPT_MOUNT)) return -BCH_ERR_option_name; - if (id == Opt_acl && - !IS_ENABLED(CONFIG_BCACHEFS_POSIX_ACL)) - return -BCH_ERR_option_name; - if ((id == Opt_usrquota || id == Opt_grpquota) && !IS_ENABLED(CONFIG_BCACHEFS_QUOTA)) diff --git a/libbcachefs/sb-errors.c b/libbcachefs/sb-errors.c index b356e801..03b2f855 100644 --- a/libbcachefs/sb-errors.c +++ b/libbcachefs/sb-errors.c @@ -70,10 +70,12 @@ static void bch2_sb_errors_to_text(struct printbuf *out, struct bch_sb *sb, struct bch_sb_field_error_entry *sorted = kvmalloc_array(nr, sizeof(*sorted), GFP_KERNEL); - if (sorted) + if (sorted) { + memcpy(sorted, e->entries, nr * sizeof(e->entries[0])); sort(sorted, nr, sizeof(*sorted), error_entry_cmp, NULL); - else + } else { sorted = e->entries; + } if (out->nr_tabstops <= 1) printbuf_tabstop_push(out, 16); diff --git a/libbcachefs/util.h b/libbcachefs/util.h index 555e0d8f..eb4222f0 100644 --- a/libbcachefs/util.h +++ b/libbcachefs/util.h @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include diff --git a/libbcachefs/vendor/min_heap.c b/libbcachefs/vendor/min_heap.c new file mode 100644 index 00000000..21f74454 --- /dev/null +++ b/libbcachefs/vendor/min_heap.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "min_heap.h" + +void __bch2_min_heap_init(min_heap_char *heap, void *data, size_t size) +{ + __min_heap_init_inline(heap, data, size); +} + +void *__bch2_min_heap_peek(struct min_heap_char *heap) +{ + return __min_heap_peek_inline(heap); +} + +bool __bch2_min_heap_full(min_heap_char *heap) +{ + return __min_heap_full_inline(heap); +} + +void __bch2_min_heap_sift_down(min_heap_char *heap, size_t pos, size_t elem_size, + const struct min_heap_callbacks *func, void *args) +{ + __min_heap_sift_down_inline(heap, pos, elem_size, func, args); +} + +void __bch2_min_heap_sift_up(min_heap_char *heap, size_t elem_size, size_t idx, + const struct min_heap_callbacks *func, void *args) +{ + __min_heap_sift_up_inline(heap, elem_size, idx, func, args); +} + +void __bch2_min_heapify_all(min_heap_char *heap, size_t elem_size, + const struct min_heap_callbacks *func, void *args) +{ + __min_heapify_all_inline(heap, elem_size, func, args); +} + +bool __bch2_min_heap_pop(min_heap_char *heap, size_t elem_size, + const struct min_heap_callbacks *func, void *args) +{ + return __min_heap_pop_inline(heap, elem_size, func, args); +} + +void __bch2_min_heap_pop_push(min_heap_char *heap, const void *element, size_t elem_size, + const struct min_heap_callbacks *func, void *args) +{ + __min_heap_pop_push_inline(heap, element, elem_size, func, args); +} + +bool __bch2_min_heap_push(min_heap_char *heap, const void *element, size_t elem_size, + const struct min_heap_callbacks *func, void *args) +{ + return __min_heap_push_inline(heap, element, elem_size, func, args); +} + +bool __bch2_min_heap_del(min_heap_char *heap, size_t elem_size, size_t idx, + const struct min_heap_callbacks *func, void *args) +{ + return __min_heap_del_inline(heap, elem_size, idx, func, args); +} diff --git a/libbcachefs/vendor/min_heap.h b/libbcachefs/vendor/min_heap.h new file mode 100644 index 00000000..865c8b33 --- /dev/null +++ b/libbcachefs/vendor/min_heap.h @@ -0,0 +1,477 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_MIN_HEAP_H +#define _LINUX_MIN_HEAP_H + +#include +#include +#include + +/* + * The Min Heap API provides utilities for managing min-heaps, a binary tree + * structure where each node's value is less than or equal to its children's + * values, ensuring the smallest element is at the root. + * + * Users should avoid directly calling functions prefixed with __min_heap_*(). + * Instead, use the provided macro wrappers. + * + * For further details and examples, refer to Documentation/core-api/min_heap.rst. + */ + +/** + * Data structure to hold a min-heap. + * @nr: Number of elements currently in the heap. + * @size: Maximum number of elements that can be held in current storage. + * @data: Pointer to the start of array holding the heap elements. + * @preallocated: Start of the static preallocated array holding the heap elements. + */ +#define MIN_HEAP_PREALLOCATED(_type, _name, _nr) \ +struct _name { \ + size_t nr; \ + size_t size; \ + _type *data; \ + _type preallocated[_nr]; \ +} + +#define DEFINE_MIN_HEAP(_type, _name) MIN_HEAP_PREALLOCATED(_type, _name, 0) + +typedef DEFINE_MIN_HEAP(char, min_heap_char) min_heap_char; + +#define __minheap_cast(_heap) (typeof((_heap)->data[0]) *) +#define __minheap_obj_size(_heap) sizeof((_heap)->data[0]) + +/** + * struct min_heap_callbacks - Data/functions to customise the min_heap. + * @less: Partial order function for this heap. + * @swp: Swap elements function. + */ +struct min_heap_callbacks { + bool (*less)(const void *lhs, const void *rhs, void *args); + void (*swp)(void *lhs, void *rhs, void *args); +}; + +/** + * is_aligned - is this pointer & size okay for word-wide copying? + * @base: pointer to data + * @size: size of each element + * @align: required alignment (typically 4 or 8) + * + * Returns true if elements can be copied using word loads and stores. + * The size must be a multiple of the alignment, and the base address must + * be if we do not have CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS. + * + * For some reason, gcc doesn't know to optimize "if (a & mask || b & mask)" + * to "if ((a | b) & mask)", so we do that by hand. + */ +__attribute_const__ __always_inline +static bool is_aligned(const void *base, size_t size, unsigned char align) +{ + unsigned char lsbits = (unsigned char)size; + + (void)base; +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + lsbits |= (unsigned char)(uintptr_t)base; +#endif + return (lsbits & (align - 1)) == 0; +} + +/** + * swap_words_32 - swap two elements in 32-bit chunks + * @a: pointer to the first element to swap + * @b: pointer to the second element to swap + * @n: element size (must be a multiple of 4) + * + * Exchange the two objects in memory. This exploits base+index addressing, + * which basically all CPUs have, to minimize loop overhead computations. + * + * For some reason, on x86 gcc 7.3.0 adds a redundant test of n at the + * bottom of the loop, even though the zero flag is still valid from the + * subtract (since the intervening mov instructions don't alter the flags). + * Gcc 8.1.0 doesn't have that problem. + */ +static __always_inline +void swap_words_32(void *a, void *b, size_t n) +{ + do { + u32 t = *(u32 *)(a + (n -= 4)); + *(u32 *)(a + n) = *(u32 *)(b + n); + *(u32 *)(b + n) = t; + } while (n); +} + +/** + * swap_words_64 - swap two elements in 64-bit chunks + * @a: pointer to the first element to swap + * @b: pointer to the second element to swap + * @n: element size (must be a multiple of 8) + * + * Exchange the two objects in memory. This exploits base+index + * addressing, which basically all CPUs have, to minimize loop overhead + * computations. + * + * We'd like to use 64-bit loads if possible. If they're not, emulating + * one requires base+index+4 addressing which x86 has but most other + * processors do not. If CONFIG_64BIT, we definitely have 64-bit loads, + * but it's possible to have 64-bit loads without 64-bit pointers (e.g. + * x32 ABI). Are there any cases the kernel needs to worry about? + */ +static __always_inline +void swap_words_64(void *a, void *b, size_t n) +{ + do { +#ifdef CONFIG_64BIT + u64 t = *(u64 *)(a + (n -= 8)); + *(u64 *)(a + n) = *(u64 *)(b + n); + *(u64 *)(b + n) = t; +#else + /* Use two 32-bit transfers to avoid base+index+4 addressing */ + u32 t = *(u32 *)(a + (n -= 4)); + *(u32 *)(a + n) = *(u32 *)(b + n); + *(u32 *)(b + n) = t; + + t = *(u32 *)(a + (n -= 4)); + *(u32 *)(a + n) = *(u32 *)(b + n); + *(u32 *)(b + n) = t; +#endif + } while (n); +} + +/** + * swap_bytes - swap two elements a byte at a time + * @a: pointer to the first element to swap + * @b: pointer to the second element to swap + * @n: element size + * + * This is the fallback if alignment doesn't allow using larger chunks. + */ +static __always_inline +void swap_bytes(void *a, void *b, size_t n) +{ + do { + char t = ((char *)a)[--n]; + ((char *)a)[n] = ((char *)b)[n]; + ((char *)b)[n] = t; + } while (n); +} + +/* + * The values are arbitrary as long as they can't be confused with + * a pointer, but small integers make for the smallest compare + * instructions. + */ +#define SWAP_WORDS_64 ((void (*)(void *, void *, void *))0) +#define SWAP_WORDS_32 ((void (*)(void *, void *, void *))1) +#define SWAP_BYTES ((void (*)(void *, void *, void *))2) + +/* + * Selects the appropriate swap function based on the element size. + */ +static __always_inline +void *select_swap_func(const void *base, size_t size) +{ + if (is_aligned(base, size, 8)) + return SWAP_WORDS_64; + else if (is_aligned(base, size, 4)) + return SWAP_WORDS_32; + else + return SWAP_BYTES; +} + +static __always_inline +void do_swap(void *a, void *b, size_t size, void (*swap_func)(void *lhs, void *rhs, void *args), + void *priv) +{ + if (swap_func == SWAP_WORDS_64) + swap_words_64(a, b, size); + else if (swap_func == SWAP_WORDS_32) + swap_words_32(a, b, size); + else if (swap_func == SWAP_BYTES) + swap_bytes(a, b, size); + else + swap_func(a, b, priv); +} + +/** + * parent - given the offset of the child, find the offset of the parent. + * @i: the offset of the heap element whose parent is sought. Non-zero. + * @lsbit: a precomputed 1-bit mask, equal to "size & -size" + * @size: size of each element + * + * In terms of array indexes, the parent of element j = @i/@size is simply + * (j-1)/2. But when working in byte offsets, we can't use implicit + * truncation of integer divides. + * + * Fortunately, we only need one bit of the quotient, not the full divide. + * @size has a least significant bit. That bit will be clear if @i is + * an even multiple of @size, and set if it's an odd multiple. + * + * Logically, we're doing "if (i & lsbit) i -= size;", but since the + * branch is unpredictable, it's done with a bit of clever branch-free + * code instead. + */ +__attribute_const__ __always_inline +static size_t parent(size_t i, unsigned int lsbit, size_t size) +{ + i -= size; + i -= size & -(i & lsbit); + return i / 2; +} + +/* Initialize a min-heap. */ +static __always_inline +void __min_heap_init_inline(min_heap_char *heap, void *data, size_t size) +{ + heap->nr = 0; + heap->size = size; + if (data) + heap->data = data; + else + heap->data = heap->preallocated; +} + +#define min_heap_init_inline(_heap, _data, _size) \ + __min_heap_init_inline(container_of(&(_heap)->nr, min_heap_char, nr), _data, _size) + +/* Get the minimum element from the heap. */ +static __always_inline +void *__min_heap_peek_inline(struct min_heap_char *heap) +{ + return heap->nr ? heap->data : NULL; +} + +#define min_heap_peek_inline(_heap) \ + (__minheap_cast(_heap) \ + __min_heap_peek_inline(container_of(&(_heap)->nr, min_heap_char, nr))) + +/* Check if the heap is full. */ +static __always_inline +bool __min_heap_full_inline(min_heap_char *heap) +{ + return heap->nr == heap->size; +} + +#define min_heap_full_inline(_heap) \ + __min_heap_full_inline(container_of(&(_heap)->nr, min_heap_char, nr)) + +/* Sift the element at pos down the heap. */ +static __always_inline +void __min_heap_sift_down_inline(min_heap_char *heap, size_t pos, size_t elem_size, + const struct min_heap_callbacks *func, void *args) +{ + const unsigned long lsbit = elem_size & -elem_size; + void *data = heap->data; + void (*swp)(void *lhs, void *rhs, void *args) = func->swp; + /* pre-scale counters for performance */ + size_t a = pos * elem_size; + size_t b, c, d; + size_t n = heap->nr * elem_size; + + if (!swp) + swp = select_swap_func(data, elem_size); + + /* Find the sift-down path all the way to the leaves. */ + for (b = a; c = 2 * b + elem_size, (d = c + elem_size) < n;) + b = func->less(data + c, data + d, args) ? c : d; + + /* Special case for the last leaf with no sibling. */ + if (d == n) + b = c; + + /* Backtrack to the correct location. */ + while (b != a && func->less(data + a, data + b, args)) + b = parent(b, lsbit, elem_size); + + /* Shift the element into its correct place. */ + c = b; + while (b != a) { + b = parent(b, lsbit, elem_size); + do_swap(data + b, data + c, elem_size, swp, args); + } +} + +#define min_heap_sift_down_inline(_heap, _pos, _func, _args) \ + __min_heap_sift_down_inline(container_of(&(_heap)->nr, min_heap_char, nr), _pos, \ + __minheap_obj_size(_heap), _func, _args) + +/* Sift up ith element from the heap, O(log2(nr)). */ +static __always_inline +void __min_heap_sift_up_inline(min_heap_char *heap, size_t elem_size, size_t idx, + const struct min_heap_callbacks *func, void *args) +{ + const unsigned long lsbit = elem_size & -elem_size; + void *data = heap->data; + void (*swp)(void *lhs, void *rhs, void *args) = func->swp; + /* pre-scale counters for performance */ + size_t a = idx * elem_size, b; + + if (!swp) + swp = select_swap_func(data, elem_size); + + while (a) { + b = parent(a, lsbit, elem_size); + if (func->less(data + b, data + a, args)) + break; + do_swap(data + a, data + b, elem_size, swp, args); + a = b; + } +} + +#define min_heap_sift_up_inline(_heap, _idx, _func, _args) \ + __min_heap_sift_up_inline(container_of(&(_heap)->nr, min_heap_char, nr), \ + __minheap_obj_size(_heap), _idx, _func, _args) + +/* Floyd's approach to heapification that is O(nr). */ +static __always_inline +void __min_heapify_all_inline(min_heap_char *heap, size_t elem_size, + const struct min_heap_callbacks *func, void *args) +{ + ssize_t i; + + for (i = heap->nr / 2 - 1; i >= 0; i--) + __min_heap_sift_down_inline(heap, i, elem_size, func, args); +} + +#define min_heapify_all_inline(_heap, _func, _args) \ + __min_heapify_all_inline(container_of(&(_heap)->nr, min_heap_char, nr), \ + __minheap_obj_size(_heap), _func, _args) + +/* Remove minimum element from the heap, O(log2(nr)). */ +static __always_inline +bool __min_heap_pop_inline(min_heap_char *heap, size_t elem_size, + const struct min_heap_callbacks *func, void *args) +{ + void *data = heap->data; + + if (WARN_ONCE(heap->nr <= 0, "Popping an empty heap")) + return false; + + /* Place last element at the root (position 0) and then sift down. */ + heap->nr--; + memcpy(data, data + (heap->nr * elem_size), elem_size); + __min_heap_sift_down_inline(heap, 0, elem_size, func, args); + + return true; +} + +#define min_heap_pop_inline(_heap, _func, _args) \ + __min_heap_pop_inline(container_of(&(_heap)->nr, min_heap_char, nr), \ + __minheap_obj_size(_heap), _func, _args) + +/* + * Remove the minimum element and then push the given element. The + * implementation performs 1 sift (O(log2(nr))) and is therefore more + * efficient than a pop followed by a push that does 2. + */ +static __always_inline +void __min_heap_pop_push_inline(min_heap_char *heap, const void *element, size_t elem_size, + const struct min_heap_callbacks *func, void *args) +{ + memcpy(heap->data, element, elem_size); + __min_heap_sift_down_inline(heap, 0, elem_size, func, args); +} + +#define min_heap_pop_push_inline(_heap, _element, _func, _args) \ + __min_heap_pop_push_inline(container_of(&(_heap)->nr, min_heap_char, nr), _element, \ + __minheap_obj_size(_heap), _func, _args) + +/* Push an element on to the heap, O(log2(nr)). */ +static __always_inline +bool __min_heap_push_inline(min_heap_char *heap, const void *element, size_t elem_size, + const struct min_heap_callbacks *func, void *args) +{ + void *data = heap->data; + size_t pos; + + if (WARN_ONCE(heap->nr >= heap->size, "Pushing on a full heap")) + return false; + + /* Place at the end of data. */ + pos = heap->nr; + memcpy(data + (pos * elem_size), element, elem_size); + heap->nr++; + + /* Sift child at pos up. */ + __min_heap_sift_up_inline(heap, elem_size, pos, func, args); + + return true; +} + +#define min_heap_push_inline(_heap, _element, _func, _args) \ + __min_heap_push_inline(container_of(&(_heap)->nr, min_heap_char, nr), _element, \ + __minheap_obj_size(_heap), _func, _args) + +/* Remove ith element from the heap, O(log2(nr)). */ +static __always_inline +bool __min_heap_del_inline(min_heap_char *heap, size_t elem_size, size_t idx, + const struct min_heap_callbacks *func, void *args) +{ + void *data = heap->data; + void (*swp)(void *lhs, void *rhs, void *args) = func->swp; + + if (WARN_ONCE(heap->nr <= 0, "Popping an empty heap")) + return false; + + if (!swp) + swp = select_swap_func(data, elem_size); + + /* Place last element at the root (position 0) and then sift down. */ + heap->nr--; + if (idx == heap->nr) + return true; + do_swap(data + (idx * elem_size), data + (heap->nr * elem_size), elem_size, swp, args); + __min_heap_sift_up_inline(heap, elem_size, idx, func, args); + __min_heap_sift_down_inline(heap, idx, elem_size, func, args); + + return true; +} + +#define min_heap_del_inline(_heap, _idx, _func, _args) \ + __min_heap_del_inline(container_of(&(_heap)->nr, min_heap_char, nr), \ + __minheap_obj_size(_heap), _idx, _func, _args) + +void __bch2_min_heap_init(min_heap_char *heap, void *data, size_t size); +void *__bch2_min_heap_peek(struct min_heap_char *heap); +bool __bch2_min_heap_full(min_heap_char *heap); +void __bch2_min_heap_sift_down(min_heap_char *heap, size_t pos, size_t elem_size, + const struct min_heap_callbacks *func, void *args); +void __bch2_min_heap_sift_up(min_heap_char *heap, size_t elem_size, size_t idx, + const struct min_heap_callbacks *func, void *args); +void __bch2_min_heapify_all(min_heap_char *heap, size_t elem_size, + const struct min_heap_callbacks *func, void *args); +bool __bch2_min_heap_pop(min_heap_char *heap, size_t elem_size, + const struct min_heap_callbacks *func, void *args); +void __bch2_min_heap_pop_push(min_heap_char *heap, const void *element, size_t elem_size, + const struct min_heap_callbacks *func, void *args); +bool __bch2_min_heap_push(min_heap_char *heap, const void *element, size_t elem_size, + const struct min_heap_callbacks *func, void *args); +bool __bch2_min_heap_del(min_heap_char *heap, size_t elem_size, size_t idx, + const struct min_heap_callbacks *func, void *args); + +#define min_heap_init(_heap, _data, _size) \ + __bch2_min_heap_init(container_of(&(_heap)->nr, min_heap_char, nr), _data, _size) +#define min_heap_peek(_heap) \ + (__minheap_cast(_heap) __bch2_min_heap_peek(container_of(&(_heap)->nr, min_heap_char, nr))) +#define min_heap_full(_heap) \ + __bch2_min_heap_full(container_of(&(_heap)->nr, min_heap_char, nr)) +#define min_heap_sift_down(_heap, _pos, _func, _args) \ + __bch2_min_heap_sift_down(container_of(&(_heap)->nr, min_heap_char, nr), _pos, \ + __minheap_obj_size(_heap), _func, _args) +#define min_heap_sift_up(_heap, _idx, _func, _args) \ + __bch2_min_heap_sift_up(container_of(&(_heap)->nr, min_heap_char, nr), \ + __minheap_obj_size(_heap), _idx, _func, _args) +#define min_heapify_all(_heap, _func, _args) \ + __bch2_min_heapify_all(container_of(&(_heap)->nr, min_heap_char, nr), \ + __minheap_obj_size(_heap), _func, _args) +#define min_heap_pop(_heap, _func, _args) \ + __bch2_min_heap_pop(container_of(&(_heap)->nr, min_heap_char, nr), \ + __minheap_obj_size(_heap), _func, _args) +#define min_heap_pop_push(_heap, _element, _func, _args) \ + __bch2_min_heap_pop_push(container_of(&(_heap)->nr, min_heap_char, nr), _element, \ + __minheap_obj_size(_heap), _func, _args) +#define min_heap_push(_heap, _element, _func, _args) \ + __bch2_min_heap_push(container_of(&(_heap)->nr, min_heap_char, nr), _element, \ + __minheap_obj_size(_heap), _func, _args) +#define min_heap_del(_heap, _idx, _func, _args) \ + __bch2_min_heap_del(container_of(&(_heap)->nr, min_heap_char, nr), \ + __minheap_obj_size(_heap), _idx, _func, _args) + +#endif /* _LINUX_MIN_HEAP_H */