From 1b74a2bba1486be1dd44f5955744ea58e056f259 Mon Sep 17 00:00:00 2001 From: Justin Husted Date: Sat, 28 Dec 2019 16:56:24 -0800 Subject: [PATCH 1/7] bcachefs-tools: fix test bug in valgrind handling. Signed-off-by: Justin Husted --- tests/util.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/util.py b/tests/util.py index c96f2eeb..c4d74310 100644 --- a/tests/util.py +++ b/tests/util.py @@ -22,10 +22,14 @@ class ValgrindFailedError(Exception): def __init__(self, log): self.log = log -def check_valgrind(logfile): - log = logfile.read().decode('utf-8') +def check_valgrind(log): m = VPAT.search(log) - assert m is not None, 'Internal error: valgrind log did not match.' + if m is None: + print('Internal error: valgrind log did not match.') + print('-- valgrind log:') + print(log) + print('-- end log --') + assert False errors = int(m.group(1)) if errors > 0: @@ -53,7 +57,7 @@ def run(cmd, *args, valgrind=False, check=False): encoding='utf-8', check=check) if valgrind: - check_valgrind(vout) + check_valgrind(vout.read().decode('utf-8')) return res @@ -150,14 +154,14 @@ class BFuse: def run(self): """Background thread which runs "bcachefs fusemount" under valgrind""" - vout = None + vlog = None cmd = [] if ENABLE_VALGRIND: - vout = tempfile.NamedTemporaryFile() + vlog = tempfile.NamedTemporaryFile() cmd += [ 'valgrind', '--leak-check=full', - '--log-file={}'.format(vout.name) ] + '--log-file={}'.format(vlog.name) ] cmd += [ BCH_PATH, 'fusemount', '-f', self.dev, self.mnt] @@ -178,7 +182,7 @@ class BFuse: self.stdout = out1 + out2 self.stderr = err.read() self.returncode = self.proc.returncode - self.vout = vout + self.vout = vlog.read().decode('utf-8') def expect(self, pipe, regex): """Wait for the child process to mount.""" @@ -208,13 +212,15 @@ class BFuse: def unmount(self, timeout=None): print("Unmounting fuse.") run("fusermount3", "-zu", self.mnt) - print("Waiting for thread to exit.") if self.thread: + print("Waiting for thread to exit.") self.thread.join(timeout) if self.thread.is_alive(): self.proc.kill() self.thread.join() + else: + print("Thread was already done.") self.thread = None self.ready.clear() From 3b8db3d7bede084f6bb242436917899a0e00519f Mon Sep 17 00:00:00 2001 From: Justin Husted Date: Sat, 28 Dec 2019 15:16:09 -0800 Subject: [PATCH 2/7] bcachefs-tools: Add a smoke test master script. This script builds and runs with various options to enable easy test coverage. Signed-off-by: Justin Husted --- Makefile | 5 +++- smoke_test | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100755 smoke_test diff --git a/Makefile b/Makefile index 2f6531b6..25de546a 100644 --- a/Makefile +++ b/Makefile @@ -71,8 +71,11 @@ endif .PHONY: all all: bcachefs +.PHONY: tests +tests: tests/test_helper + .PHONY: check -check: tests/test_helper bcachefs +check: tests bcachefs cd tests; $(PYTEST) .PHONY: TAGS tags diff --git a/smoke_test b/smoke_test new file mode 100755 index 00000000..076806d8 --- /dev/null +++ b/smoke_test @@ -0,0 +1,82 @@ +#!/bin/bash +# +# This is a smoke test of bcachefs-tools. +# +# It builds the source with multiple options (debug, release, valgrind, FUSE) +# and runs the test suite. +# +# Returns 0 on success, nonzero on any failure. +# +# Dependencies: +# +# valgrind, python3-pytest, python3-pytest-xdist +# +# On debian/ubuntu based systems, install with: +# +# apt install valgrind python3-pytest python3-pytest-xdist +# +# You also currently need fuse 3.7 or later. Fuse 3.7 unfortunately requires +# debian sid or bullseye at this time, so you may need to install from source. + +set -e + +spam=$(tempfile) +unset BCACHEFS_FUSE BCACHEFS_TEST_USE_VALGRIND D + +trap "set +x; cat ${spam}; rm -f ${spam} ; echo; echo FAILED." EXIT + +echo -- Verify dependencies -- +pkg-config --atleast-version 3.7.0 fuse3 +python3 -c "import pytest" +python3 -c "import xdist" +which valgrind > /dev/null +echo OK + +JOBS=$(nproc) +function build() { + echo Building. + make -j ${JOBS} clean > ${spam} 2>&1 + make -j ${JOBS} tests bcachefs > ${spam} 2>&1 + truncate -s0 ${spam} +} + +function test() { + echo Running tests. + ( + cd tests + pytest-3 -n${JOBS} + ) > ${spam} 2>&1 +} + +function test_vg() { + echo Running tests with valgrind. + ( + export BCACHEFS_TEST_USE_VALGRIND=yes + cd tests + pytest-3 -n${JOBS} + ) > ${spam} 2>&1 +} + + +echo -- Test: default -- +build +test + +echo -- Test: debug -- +export D=1 +build +test + +echo -- Test: debug with valgrind -- +test_vg + +echo -- Test: fuse debug -- +export BCACHEFS_FUSE=1 +build +test + +echo -- Test: fuse debug with valgrind -- +test_vg + +rm -f ${spam} +trap "set +x; echo; echo SUCCESS." EXIT From 5d79efdd199ab8018f53bcebb4e04405c080ce01 Mon Sep 17 00:00:00 2001 From: Justin Husted Date: Sat, 28 Dec 2019 20:09:22 -0800 Subject: [PATCH 3/7] bcachefs-tools: Fix test. Signed-off-by: Justin Husted --- tests/test_fuse.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_fuse.py b/tests/test_fuse.py index c7608f4c..660d92de 100644 --- a/tests/test_fuse.py +++ b/tests/test_fuse.py @@ -230,3 +230,6 @@ def test_write(bfuse): assert ts.contains(post_st.st_mtime) assert path.read_bytes() == b'test' + + bfuse.unmount() + bfuse.verify() From 15bcfb2a33cc3843c9b90b1370340aa615054703 Mon Sep 17 00:00:00 2001 From: Justin Husted Date: Sat, 28 Dec 2019 18:50:49 -0800 Subject: [PATCH 4/7] Add a travis.yml file for bcachefs-tools. Signed-off-by: Justin Husted --- .gitignore | 1 + .travis.yml | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 .travis.yml diff --git a/.gitignore b/.gitignore index 2e06ef53..a19453e9 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ tests/__pycache__/ # dot-files that we don't want to ignore !.gitignore +!.travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..de845ba4 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,32 @@ +os: linux +dist: bionic +language: c +arch: amd64 +addons: + apt: + packages: + - valgrind + - python3-pytest + - python3-pytest-xdist + - meson + - ninja-build + - pkg-config + - libaio-dev + - libblkid-dev + - libkeyutils-dev + - liblz4-dev + - libscrypt-dev + - libsodium-dev + - liburcu-dev + - libzstd-dev + - uuid-dev + - zlib1g-dev + +before_install: + - wget https://github.com/libfuse/libfuse/archive/fuse-3.7.0.tar.gz -O /tmp/fuse.tar.gz + - tar -C /tmp -zxvf /tmp/fuse.tar.gz + - mkdir /tmp/libfuse-fuse-3.7.0/build + - pushd /tmp/libfuse-fuse-3.7.0/build && meson .. && ninja && sudo ninja install && popd + - sudo ldconfig + +script: ./smoke_test From e19950a252b8920c80d29e8364246297b24e29c2 Mon Sep 17 00:00:00 2001 From: Justin Husted Date: Sat, 28 Dec 2019 20:34:31 -0800 Subject: [PATCH 5/7] bcachefs-tools: Add arm64 build on travis-ci. Signed-off-by: Justin Husted --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index de845ba4..06835cf8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,10 @@ os: linux dist: bionic language: c -arch: amd64 +arch: + - amd64 + - arm64 + addons: apt: packages: From f24bf2af6ba91585713f9dddc59bd16e23a1ddc5 Mon Sep 17 00:00:00 2001 From: Justin Husted Date: Sat, 28 Dec 2019 20:38:26 -0800 Subject: [PATCH 6/7] Well, arm64 crashed. Turn that off for now. Signed-off-by: Justin Husted --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 06835cf8..5f347ba4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ dist: bionic language: c arch: - amd64 - - arm64 +# - arm64 addons: apt: From c54ea514161689e33be8397ddecc686d81ae2ca7 Mon Sep 17 00:00:00 2001 From: Justin Husted Date: Sun, 29 Dec 2019 03:07:56 -0800 Subject: [PATCH 7/7] Make die exit using _exit Exiting the standard way on error caused teardown in bad state problems. Signed-off-by: Justin Husted --- tools-util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools-util.c b/tools-util.c index e4d2beae..f7ca5980 100644 --- a/tools-util.c +++ b/tools-util.c @@ -31,7 +31,7 @@ void die(const char *fmt, ...) va_end(args); fputc('\n', stderr); - exit(EXIT_FAILURE); + _exit(EXIT_FAILURE); } char *mprintf(const char *fmt, ...)