From 07bf2c16cbcb97bf65fc105893d62b15bf147a0c Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Sat, 4 Oct 2025 18:29:15 +0300 Subject: [PATCH] Deb PPA rewrite, adding packages for ppc64el TLDR: instead of running the entirety of the job in the container, start a detached container, and manually run each step in detached container. This allows to run stuff outside of container. --- .github/workflows/deb-buildd.yml | 310 ++++++++++++++++++------- .github/workflows/deb-orchestrator.yml | 17 +- .github/workflows/deb-publish.yml | 236 +++++++++++++------ .github/workflows/deb-reprotest.yml | 205 ++++++++++++---- .github/workflows/deb-src.yml | 210 +++++++++++------ debian/control | 3 +- 6 files changed, 712 insertions(+), 269 deletions(-) diff --git a/.github/workflows/deb-buildd.yml b/.github/workflows/deb-buildd.yml index db171a1c..73667f0b 100644 --- a/.github/workflows/deb-buildd.yml +++ b/.github/workflows/deb-buildd.yml @@ -7,7 +7,13 @@ on: runs-on: required: true type: string - arch: + machine-arch: + required: true + type: string + build-arch: + required: true + type: string + host-arch: required: true type: string dist-name: @@ -23,45 +29,94 @@ on: jobs: linux: runs-on: ${{ inputs.runs-on }} - container: - image: debian:unstable-slim - options: --privileged --cap-add=SYS_ADMIN --security-opt=apparmor:unconfined --tmpfs /tmp:exec --tmpfs /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} - env: - DEBFULLNAME: apt.bcachefs.org CI bot - DEBEMAIL: linux-bcachefs@vger.kernel.org - DEB_SIGN_KEYID: "${{ secrets.GPG_SIGNING_SUBKEY_FINGERPRINT }}" permissions: id-token: write contents: read attestations: write steps: - - name: Install necessary packages - timeout-minutes: 2 + - name: Configure baseline system + timeout-minutes: 1 + shell: sudo sh "{0}" run: | set -xe - tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io > /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null <> $GITHUB_ENV + BUILD_DIR="${{ github.workspace }}/deb-bin/${{ inputs.dist-version }}/${{ inputs.host-arch }}" + echo "BUILD_DIR=$(echo ${BUILD_DIR})" >> ~/.env mkdir -p "$BUILD_DIR" tee -a ~/.sbuildrc > /dev/null < /dev/null < [ "export LD_PRELOAD=/usr/lib/$(dpkg-architecture -q DEB_BUILD_GNU_TYPE)/libeatmydata.so" ] + 'chroot-setup-commands' => [ + "export LD_PRELOAD=/usr/lib/$(dpkg-architecture -a "${{ inputs.build-arch }}" -q DEB_HOST_MULTIARCH)/libeatmydata.so" + ] }; \$build_environment = { - "LD_PRELOAD" => "/usr/lib/$(dpkg-architecture -q DEB_BUILD_GNU_TYPE)/libeatmydata.so" + "LD_PRELOAD" => + "/usr/lib/$(dpkg-architecture -a "${{ inputs.build-arch }}" -q DEB_HOST_MULTIARCH)/libeatmydata.so" }; - \$environment_filter = [Dpkg::BuildInfo::get_build_env_allowed(), '^LD_PRELOAD$']; + \$environment_filter = [ + Dpkg::BuildInfo::get_build_env_allowed(), + '^LD_PRELOAD$' + ]; push @{\$unshare_mmdebstrap_extra_args}, "*", [ '--hook-dir=/usr/share/mmdebstrap/hooks/eatmydata' ]; @@ -171,10 +247,10 @@ jobs: if [ "${{ inputs.dist-name }}" = "debian" ]; then MIRROR="http://deb.debian.org/debian" - elif [ "${{ inputs.dist-name }}" = "ubuntu" ] && [ "${{ inputs.arch }}" = "amd64" ]; + elif [ "${{ inputs.dist-name }}" = "ubuntu" ] && [ "${{ inputs.build-arch }}" = "amd64" ]; then MIRROR="http://archive.ubuntu.com/ubuntu" - elif [ "${{ inputs.dist-name }}" = "ubuntu" ] && [ "${{ inputs.arch }}" != "amd64" ]; + elif [ "${{ inputs.dist-name }}" = "ubuntu" ] && [ "${{ inputs.build-arch }}" != "amd64" ]; then MIRROR="http://ports.ubuntu.com/ubuntu-ports" else @@ -189,7 +265,7 @@ jobs: else exit 1 fi - echo "MIRROR=$(echo ${MIRROR})" >> $GITHUB_ENV + echo "MIRROR=$(echo ${MIRROR})" >> ~/.env if [ "${{ inputs.dist-name }}" = "debian" ]; then tee -a ~/.sbuildrc > /dev/null <> $GITHUB_ENV - eatmydata mmdebstrap-autopkgtest-build-qemu --boot=efi --mirror "${{ env.MIRROR }}" "${{ inputs.dist-version }}" "$IMAGE" + echo "IMAGE=$(echo ${IMAGE})" >> ~/.env + mmdebstrap-autopkgtest-build-qemu --boot=efi --mirror "$MIRROR" "${{ inputs.dist-version }}" "$IMAGE" - name: Run autopkgtest timeout-minutes: 10 if: steps.qemu-image.conclusion != 'skipped' + shell: sudo podman exec --interactive --tty container eatmydata sh "{0}" run: | set -xe - eatmydata autopkgtest --no-built-binaries --apt-upgrade "$GITHUB_WORKSPACE/deb-src"/*.dsc "${{ env.BUILD_DIR }}"/*.changes -- autopkgtest-virt-qemu --qemu-options="-smp $(nproc --all)" --boot=efi "${{ env.IMAGE }}" + . ~/.env + if [ "${{ inputs.machine-arch }}" = "amd64" ]; + then + QEMU_OPTIONS="-enable-kvm -cpu host" + elif [ "${{ inputs.machine-arch }}" = "arm64" ]; + then + QEMU_OPTIONS="-machine virt -cpu cortex-a710 -accel tcg" + else + exit 1 + fi + autopkgtest \ + --no-built-binaries \ + --setup-commands "apt update && apt install openssl" \ + --apt-upgrade \ + "${{ github.workspace }}/deb-src"/*.dsc \ + "$BUILD_DIR"/*.changes \ + -- \ + autopkgtest-virt-qemu \ + --cpus=$(nproc --all) \ + --qemu-options="$QEMU_OPTIONS" \ + --boot=efi \ + "$IMAGE" \ + ; diff --git a/.github/workflows/deb-orchestrator.yml b/.github/workflows/deb-orchestrator.yml index 683f244f..5e69dbd1 100644 --- a/.github/workflows/deb-orchestrator.yml +++ b/.github/workflows/deb-orchestrator.yml @@ -22,9 +22,7 @@ jobs: attestations: write uses: ./.github/workflows/deb-src.yml with: - runs-on: ubuntu-latest - dist-name: debian - dist-version: unstable + runs-on: ubuntu-24.04 secrets: GPG_SECRET_SUBKEYS: ${{ secrets.GPG_SECRET_SUBKEYS }} GPG_SIGNING_SUBKEY_FINGERPRINT: ${{ secrets.GPG_SIGNING_SUBKEY_FINGERPRINT }} @@ -38,8 +36,9 @@ jobs: fail-fast: false matrix: stack: - - { runs-on: "ubuntu-latest", arch: "amd64" } - - { runs-on: "ubuntu-24.04-arm", arch: "arm64" } + - { runs-on: "ubuntu-24.04", machine-arch: "amd64", build-arch: "amd64", host-arch: "amd64" } + - { runs-on: "ubuntu-24.04", machine-arch: "amd64", build-arch: "ppc64el", host-arch: "ppc64el" } + - { runs-on: "ubuntu-24.04-arm", machine-arch: "arm64", build-arch: "arm64", host-arch: "arm64" } dist: - { name: debian, version: unstable } - { name: debian, version: forky } # Debian 14 @@ -50,7 +49,9 @@ jobs: with: deb-src-artifact-id: ${{ needs.source-only.outputs.deb-src-artifact-id }} runs-on: ${{ matrix.stack.runs-on }} - arch: ${{ matrix.stack.arch }} + machine-arch: ${{ matrix.stack.machine-arch }} + build-arch: ${{ matrix.stack.build-arch }} + host-arch: ${{ matrix.stack.host-arch }} dist-name: ${{ matrix.dist.name }} dist-version: ${{ matrix.dist.version }} secrets: @@ -61,7 +62,7 @@ jobs: uses: ./.github/workflows/deb-reprotest.yml with: deb-src-artifact-id: ${{ needs.source-only.outputs.deb-src-artifact-id }} - runs-on: "ubuntu-latest" + runs-on: "ubuntu-24.04" arch: amd64 dist-name: debian dist-version: unstable @@ -72,6 +73,8 @@ jobs: needs: [ source-only, buildd, reprotest ] if: github.event_name != 'pull_request' && (github.ref_type == 'tag' || (github.ref_type == 'branch' && github.ref_name == 'master')) uses: ./.github/workflows/deb-publish.yml + with: + runs-on: ubuntu-24.04 secrets: GPG_SECRET_SUBKEYS: ${{ secrets.GPG_SECRET_SUBKEYS }} GPG_SIGNING_SUBKEY_FINGERPRINT: ${{ secrets.GPG_SIGNING_SUBKEY_FINGERPRINT }} diff --git a/.github/workflows/deb-publish.yml b/.github/workflows/deb-publish.yml index 6caf8f5a..6a7eebf6 100644 --- a/.github/workflows/deb-publish.yml +++ b/.github/workflows/deb-publish.yml @@ -1,5 +1,9 @@ on: workflow_call: + inputs: + runs-on: + required: true + type: string secrets: GPG_SECRET_SUBKEYS: required: true @@ -14,49 +18,98 @@ on: jobs: linux: - concurrency: apt.bcachefs.org - runs-on: ubuntu-latest - container: - image: debian:unstable-slim - options: --cap-add=SYS_ADMIN --security-opt=apparmor:unconfined --device /dev/fuse --tmpfs /tmp --tmpfs /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} + runs-on: ${{ inputs.runs-on }} env: SUITE: ${{ (github.event_name == 'push' && github.ref_type == 'tag') && 'release' || 'snapshot' }} + permissions: + id-token: write + contents: read + attestations: write steps: - - name: Install necessary packages - timeout-minutes: 2 + - name: Configure baseline system + timeout-minutes: 1 + shell: sudo sh "{0}" run: | set -xe - tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io > /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null <> $GITHUB_ENV + echo "SSH_AUTH_SOCK=$(echo ${SSH_AUTH_SOCK})" >> ~/.env echo "" >> /etc/ssh/ssh_known_hosts echo "${{ secrets.SSH_SERVER_KEYS }}" >> /etc/ssh/ssh_known_hosts - - name: Import GPG key + - name: Import/Configure GPG timeout-minutes: 1 id: gpg if: github.event_name != 'pull_request' - uses: crazy-max/ghaction-import-gpg@e89d40939c28e39f97cf32126055eeae86ba74ec # v6.3.0 - with: - gpg_private_key: ${{ secrets.GPG_SECRET_SUBKEYS }} - fingerprint: ${{ secrets.GPG_SIGNING_SUBKEY_FINGERPRINT }} - trust_level: 5 - - name: Finish configuring gpg-agent / ssh - timeout-minutes: 1 - if: steps.gpg.conclusion != 'skipped' + shell: sudo podman exec --interactive --tty container eatmydata sh "{0}" run: | set -xe - gpg-connect-agent 'keyattr ${{ secrets.GPG_AUTH_SUBKEY_KEYGRIP }} Use-for-ssh: true' /bye - - name: Configure GPG - timeout-minutes: 1 - if: steps.gpg.conclusion != 'skipped' - run: | - set -xe - gpg --output /etc/apt/trusted.gpg.d/apt.bcachefs.org.asc --armor --export ${{ secrets.GPG_SIGNING_SUBKEY_FINGERPRINT }} + gpg --import < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < [ "export LD_PRELOAD=/usr/lib/$(dpkg-architecture -q DEB_BUILD_GNU_TYPE)/libeatmydata.so" ] + 'chroot-setup-commands' => [ + "export LD_PRELOAD=/usr/lib/$(dpkg-architecture -q DEB_BUILD_GNU_TYPE)/libeatmydata.so" + ] }; \$build_environment = { - "LD_PRELOAD" => "/usr/lib/$(dpkg-architecture -q DEB_BUILD_GNU_TYPE)/libeatmydata.so" + "LD_PRELOAD" => + "/usr/lib/$(dpkg-architecture -q DEB_BUILD_GNU_TYPE)/libeatmydata.so" }; - \$environment_filter = [Dpkg::BuildInfo::get_build_env_allowed(), '^LD_PRELOAD$']; + \$environment_filter = [ + Dpkg::BuildInfo::get_build_env_allowed(), + '^LD_PRELOAD$' + ]; push @{\$unshare_mmdebstrap_extra_args}, "*", [ '--include=eatmydata', '--hook-dir=/usr/share/mmdebstrap/hooks/eatmydata', @@ -148,10 +204,11 @@ jobs: EOT - name: Install Rust / cargo timeout-minutes: 1 + shell: sudo podman exec --interactive --tty container eatmydata sh "{0}" run: | set -xe - eatmydata curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ - eatmydata sh -s -- --default-toolchain $RUST_VERSION --profile minimal -y + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- --default-toolchain ${{ env.RUST_VERSION }} --profile minimal -y . "$HOME/.cargo/env" - name: Fetch our git repository timeout-minutes: 1 @@ -162,11 +219,15 @@ jobs: fetch-tags: true - name: Update package version/changelog timeout-minutes: 1 + shell: sudo podman exec --interactive --tty container eatmydata sh "{0}" run: | set -xe - git config --global user.email "${{ env.DEBFULLNAME }}" - git config --global user.name "${{ env.DEBEMAIL }}" - cd "$GITHUB_WORKSPACE/bcachefs-tools" + export DEBFULLNAME="${{ env.DEBFULLNAME }}" + export DEBEMAIL="${{ env.DEBEMAIL }}" + git config --global user.email "${DEBFULLNAME}" + git config --global user.name "${DEBEMAIL}" + cd "${{ github.workspace }}/bcachefs-tools" + git config --global --add safe.directory . CURR_TAG="$(git describe --abbrev=0 --tags $(git rev-list HEAD --tags --skip=0 --max-count=1))" PREV_TAG="$(git describe --abbrev=0 --tags $(git rev-list HEAD --tags --skip=1 --max-count=1))" NEW_VERSION="${{ env.DEBPKG_EPOCH }}:$(echo $CURR_TAG | sed 's/^v//')" @@ -179,32 +240,51 @@ jobs: fi - name: Build the source-only .deb package timeout-minutes: 10 + shell: sudo podman exec --interactive --tty container eatmydata sh "{0}" run: | set -xe . "$HOME/.cargo/env" - cd "$GITHUB_WORKSPACE/bcachefs-tools" - mkdir -p "$GITHUB_WORKSPACE/deb-src" + export DEBFULLNAME="${{ env.DEBFULLNAME }}" + export DEBEMAIL="${{ env.DEBEMAIL }}" + cd "${{ github.workspace }}/bcachefs-tools" + mkdir -p "${{ github.workspace }}/deb-src" # FIXME: pubkey is not avaliable in chroot, .dsc signature verification fails - eatmydata gbp buildpackage --git-verbose --git-ignore-branch --no-clean --git-dist=${{ inputs.dist-version }} --git-builder=sbuild --source --source-only-changes --no-arch-all --no-arch-any + echo '::group::Building the package' + gbp buildpackage \ + --git-verbose \ + --git-ignore-branch \ + --no-clean \ + --git-dist=${{ inputs.dist-version }} \ + --git-builder=sbuild \ + --source \ + --source-only-changes \ + --no-arch-all \ + --no-arch-any \ + ; + echo '::endgroup::' - name: Ensure that all source artifacts are signed, or sign them timeout-minutes: 1 if: steps.gpg.conclusion != 'skipped' + shell: sudo podman exec --interactive --tty container eatmydata sh "{0}" run: | set -xe - cd "$GITHUB_WORKSPACE/deb-src" - find . -type f -not -iname '*.sig' -print0 | xargs --null -I'{}' sh -c "\ - echo 'Processing {}' && ( \ - gpg --verbose --no-default-keyring --keyring ~/.gnupg/trustedkeys.gpg --verify {} \ - || gpg --verbose --no-default-keyring --keyring ~/.gnupg/trustedkeys.gpg --verify {}.sig \ - || gpg --verbose --detach-sign {} \ - ) \ + cd "${{ github.workspace }}/deb-src" + find . -type f -not -iname '*.sig' -print0 | xargs --null -I'{}' sh -c " \ + echo '::group::Signing {}' && \ + ( \ + gpg --verbose --no-default-keyring --keyring ~/.gnupg/trustedkeys.gpg --verify {} || \ + gpg --verbose --no-default-keyring --keyring ~/.gnupg/trustedkeys.gpg --verify {}.sig || \ + gpg --verbose --detach-sign {} \ + ) && \ + echo '::endgroup::' \ " - name: Archive source build artifacts timeout-minutes: 1 + shell: sudo podman exec --interactive --tty container eatmydata sh "{0}" run: | set -xe - cd "$GITHUB_WORKSPACE/deb-src" - tar -cf "$GITHUB_WORKSPACE/deb-src/artifact-src.tar" * + cd "${{ github.workspace }}/deb-src" + tar -cf "${{ github.workspace }}/deb-src/artifact-src.tar" * - name: Attest the source-only .deb package artifact timeout-minutes: 1 if: github.event_name != 'pull_request' diff --git a/debian/control b/debian/control index 0345f84a..794ab9dc 100644 --- a/debian/control +++ b/debian/control @@ -56,7 +56,8 @@ Depends: ${misc:Depends}, initramfs-tools | linux-initramfs-tool, linux-headers-generic (>= 6.16) | pve-headers (>= 6.16) [amd64] | linux-headers-amd64 (>= 6.16) [amd64] | linux-headers-cloud-amd64 (>= 6.16) [amd64] | linux-headers-rt-amd64 (>= 6.16) [amd64] | - linux-headers-arm64 (>= 6.16) [arm64] | linux-headers-cloud-arm64 (>= 6.16) [arm64] | linux-headers-rt-arm64 (>= 6.16) [arm64] + linux-headers-arm64 (>= 6.16) [arm64] | linux-headers-cloud-arm64 (>= 6.16) [arm64] | linux-headers-rt-arm64 (>= 6.16) [arm64] | + linux-headers-ppc64el (>= 6.16) [ppc64el] | linux-headers-cloud-ppc64el (>= 6.16) [ppc64el] | linux-headers-rt-ppc64el (>= 6.16) [ppc64el] Pre-Depends: bcachefs-tools (= ${binary:Version}), Provides: bcachefs-kernel Description: bcachefs kernel module DKMS source