Migrate tool fixes

Migrate now works: there is an inconsequential free space inconsistency
after marking the new superblock (in migrate_superblock), which we're
hacking around with a fsck.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

c_src/cmd_format.c

@@ -121,7 +121,7 @@ void build_fs(struct bch_fs *c, const char *src_path)
 	if (!S_ISDIR(stat.st_mode))
 		die("%s is not a directory", src_path);
 
-	copy_fs(c, src_fd, src_path, &s);
+	copy_fs(c, src_fd, src_path, &s, 0);
 }
 
 int cmd_format(int argc, char *argv[])

c_src/cmd_migrate.c

@@ -252,16 +252,14 @@ static int migrate_fs(const char		*fs_path,
 
 	free(sb);
 
-	struct bch_opts opts = bch2_opts_empty();
-	struct bch_fs *c = NULL;
 	char *path[1] = { dev->path };
 
+	struct bch_opts opts = bch2_opts_empty();
 	opt_set(opts, sb,	sb_offset);
 	opt_set(opts, nostart,	true);
 	opt_set(opts, noexcl,	true);
-	opt_set(opts, nostart, true);
 
-	c = bch2_fs_open(path, 1, opts);
+	struct bch_fs *c = bch2_fs_open(path, 1, opts);
 	if (IS_ERR(c))
 		die("Error opening new filesystem: %s", bch2_err_str(PTR_ERR(c)));
 
@@ -269,10 +267,6 @@ static int migrate_fs(const char		*fs_path,
 	if (ret)
 		die("Error allocating buckets_nouse: %s", bch2_err_str(ret));
 
-	ret = bch2_fs_start(c);
-	if (IS_ERR(c))
-		die("Error starting new filesystem: %s", bch2_err_str(ret));
-
 	mark_unreserved_space(c, extents);
 
 	ret = bch2_fs_start(c);
@@ -286,7 +280,10 @@ static int migrate_fs(const char		*fs_path,
 		.type		= BCH_MIGRATE_migrate,
 	};
 
-	copy_fs(c, fs_fd, fs_path, &s);
+	u64 reserve_start = round_up((format_opts.superblock_size * 2 + 8) << 9,
+				     dev->opts.bucket_size);
+
+	copy_fs(c, fs_fd, fs_path, &s, reserve_start);
 
 	bch2_fs_stop(c);
 
@@ -378,7 +375,7 @@ static void migrate_superblock_usage(void)
 int cmd_migrate_superblock(int argc, char *argv[])
 {
 	char *dev = NULL;
-	u64 offset = 0;
+	u64 sb_offset = 0;
 	int opt, ret;
 
 	while ((opt = getopt(argc, argv, "d:o:h")) != -1)
@@ -387,7 +384,7 @@ int cmd_migrate_superblock(int argc, char *argv[])
 				dev = optarg;
 				break;
 			case 'o':
-				ret = kstrtou64(optarg, 10, &offset);
+				ret = kstrtou64(optarg, 10, &sb_offset);
 				if (ret)
 					die("Invalid offset");
 				break;
@@ -399,29 +396,67 @@ int cmd_migrate_superblock(int argc, char *argv[])
 	if (!dev)
 		die("Please specify a device");
 
-	if (!offset)
+	if (!sb_offset)
 		die("Please specify offset of existing superblock");
 
 	int fd = xopen(dev, O_RDWR);
-	struct bch_sb *sb = __bch2_super_read(fd, offset);
+	struct bch_sb *sb = __bch2_super_read(fd, sb_offset);
+	unsigned sb_size = 1U << sb->layout.sb_max_size_bits;
 
 	if (sb->layout.nr_superblocks >= ARRAY_SIZE(sb->layout.sb_offset))
 		die("Can't add superblock: no space left in superblock layout");
 
-	unsigned i;
+	for (unsigned i = 0; i < sb->layout.nr_superblocks; i++)
-	for (i = 0; i < sb->layout.nr_superblocks; i++)
+		if (le64_to_cpu(sb->layout.sb_offset[i]) == BCH_SB_SECTOR ||
-		if (le64_to_cpu(sb->layout.sb_offset[i]) == BCH_SB_SECTOR)
+		    le64_to_cpu(sb->layout.sb_offset[i]) == BCH_SB_SECTOR + sb_size)
-			die("Superblock layout already has default superblock");
+			die("Superblock layout already has default superblocks");
 
-	memmove(&sb->layout.sb_offset[1],
+	memmove(&sb->layout.sb_offset[2],
 		&sb->layout.sb_offset[0],
 		sb->layout.nr_superblocks * sizeof(u64));
-	sb->layout.nr_superblocks++;
+	sb->layout.nr_superblocks += 2;
-
 	sb->layout.sb_offset[0] = cpu_to_le64(BCH_SB_SECTOR);
+	sb->layout.sb_offset[1] = cpu_to_le64(BCH_SB_SECTOR + sb_size);
 
 	bch2_super_write(fd, sb);
 	close(fd);
 
+	/* mark new superblocks */
+
+	struct bch_opts opts = bch2_opts_empty();
+	opt_set(opts, nostart,	true);
+	opt_set(opts, sb,	sb_offset);
+
+	struct bch_fs *c = bch2_fs_open(&dev, 1, opts);
+	ret =   PTR_ERR_OR_ZERO(c) ?:
+		bch2_buckets_nouse_alloc(c);
+	if (ret)
+		die("error opening filesystem: %s", bch2_err_str(ret));
+
+	struct bch_dev *ca = c->devs[0];
+	for (u64 b = 0; bucket_to_sector(ca, b) < BCH_SB_SECTOR + sb_size * 2; b++)
+		set_bit(b, ca->buckets_nouse);
+
+	ret = bch2_fs_start(c);
+	if (ret)
+		die("Error starting filesystem: %s", bch2_err_str(ret));
+
+	bch2_fs_stop(c);
+
+	opts = bch2_opts_empty();
+	opt_set(opts, fsck, true);
+	opt_set(opts, fix_errors, true);
+
+	/*
+	 * Hack: the free space counters are coming out wrong after marking the
+	 * new superblock, but it's just the device counters so it's
+	 * inconsequential:
+	 */
+
+	c = bch2_fs_open(&dev, 1, opts);
+	ret =   PTR_ERR_OR_ZERO(c);
+	if (ret)
+		die("error opening filesystem: %s", bch2_err_str(ret));
+	bch2_fs_stop(c);
 	return 0;
 }

c_src/posix_to_bcachefs.c

@@ -264,7 +264,8 @@ void copy_link(struct bch_fs *c, struct bch_inode_unpacked *dst,
 
 static void copy_file(struct bch_fs *c, struct bch_inode_unpacked *dst,
 		      int src_fd, u64 src_size,
-		      char *src_path, struct copy_fs_state *s)
+		      char *src_path, struct copy_fs_state *s,
+		      u64 reserve_start)
 {
 	struct fiemap_iter iter;
 	struct fiemap_extent e;
@@ -295,11 +296,8 @@ static void copy_file(struct bch_fs *c, struct bch_inode_unpacked *dst,
 			continue;
 		}
 
-		/*
+		/* If the data is in bcachefs's superblock region, copy it: */
-		 * if the data is below 1 MB, copy it so it doesn't conflict
+		if (e.fe_physical < reserve_start) {
-		 * with bcachefs's potentially larger superblock:
-		 */
-		if (e.fe_physical < 1 << 20) {
 			copy_data(c, dst, src_fd, e.fe_logical,
 				  e.fe_logical + min(src_size - e.fe_logical,
 				      e.fe_length));
@@ -318,7 +316,8 @@ static void copy_file(struct bch_fs *c, struct bch_inode_unpacked *dst,
 static void copy_dir(struct copy_fs_state *s,
 		     struct bch_fs *c,
 		     struct bch_inode_unpacked *dst,
-		     int src_fd, const char *src_path)
+		     int src_fd, const char *src_path,
+		     u64 reserve_start)
 {
 	DIR *dir = fdopendir(src_fd);
 	struct dirent *d;
@@ -369,7 +368,7 @@ static void copy_dir(struct copy_fs_state *s,
 		switch (mode_to_type(stat.st_mode)) {
 		case DT_DIR:
 			fd = xopen(d->d_name, O_RDONLY|O_NOATIME);
-			copy_dir(s, c, &inode, fd, child_path);
+			copy_dir(s, c, &inode, fd, child_path, reserve_start);
 			close(fd);
 			break;
 		case DT_REG:
@@ -377,7 +376,7 @@ static void copy_dir(struct copy_fs_state *s,
 
 			fd = xopen(d->d_name, O_RDONLY|O_NOATIME);
 			copy_file(c, &inode, fd, stat.st_size,
-				  child_path, s);
+				  child_path, s, reserve_start);
 			close(fd);
 			break;
 		case DT_LNK:
@@ -409,7 +408,8 @@ next:
 
 static void reserve_old_fs_space(struct bch_fs *c,
 				 struct bch_inode_unpacked *root_inode,
-				 ranges *extents)
+				 ranges *extents,
+				 u64 reserve_start)
 {
 	struct bch_dev *ca = c->devs[0];
 	struct bch_inode_unpacked dst;
@@ -422,14 +422,20 @@ static void reserve_old_fs_space(struct bch_fs *c,
 
 	ranges_sort_merge(extents);
 
-	for_each_hole(iter, *extents, bucket_to_sector(ca, ca->mi.nbuckets) << 9, i)
+	for_each_hole(iter, *extents, bucket_to_sector(ca, ca->mi.nbuckets) << 9, i) {
-		link_data(c, &dst, i.start, i.start, i.end - i.start);
+		if (i.end <= reserve_start)
+			continue;
+
+		u64 start = max(i.start, reserve_start);
+
+		link_data(c, &dst, start, start, i.end - start);
+	}
 
 	update_inode(c, &dst);
 }
 
 void copy_fs(struct bch_fs *c, int src_fd, const char *src_path,
-		    struct copy_fs_state *s)
+	     struct copy_fs_state *s, u64 reserve_start)
 {
 	syncfs(src_fd);
 
@@ -448,10 +454,10 @@ void copy_fs(struct bch_fs *c, int src_fd, const char *src_path,
 
 
 	/* now, copy: */
-	copy_dir(s, c, &root_inode, src_fd, src_path);
+	copy_dir(s, c, &root_inode, src_fd, src_path, reserve_start);
 
 	if (BCH_MIGRATE_migrate == s->type)
-		reserve_old_fs_space(c, &root_inode, &s->extents);
+		reserve_old_fs_space(c, &root_inode, &s->extents, reserve_start);
 
 	update_inode(c, &root_inode);
 

c_src/posix_to_bcachefs.h

@@ -50,5 +50,5 @@ struct copy_fs_state {
  * initialized (`hardlinks` is initialized with zeroes).
  */
 void copy_fs(struct bch_fs *c, int src_fd, const char *src_path,
-		    struct copy_fs_state *s);
+		    struct copy_fs_state *s, u64);
 #endif /* _LIBBCACHE_H */