diff --git a/.github/actions/build-onedir-pkg/action.yml b/.github/actions/build-onedir-pkg/action.yml index 952d4cb05b1..0c71440bfd4 100644 --- a/.github/actions/build-onedir-pkg/action.yml +++ b/.github/actions/build-onedir-pkg/action.yml @@ -28,6 +28,11 @@ runs: path: artifacts/${{ inputs.package-name }} key: ${{ env.CACHE_SEED }}|relenv|${{ env.RELENV_VERSION }}|deps|${{ inputs.platform }}|${{ inputs.arch }}|${{ inputs.package-name }}|${{ hashFiles(format('{0}/.relenv/**/*.xz', github.workspace), 'requirements/static/pkg/*/*.txt') }} + - name: Create Source Tarball + shell: bash + run: | + nox -e build + - name: Install Salt Into Onedir if: ${{ inputs.platform != 'windows' }} env: @@ -37,7 +42,7 @@ runs: RELENV_PIP_DIR: "1" shell: bash run: | - artifacts/${{ inputs.package-name }}/bin/python3 -m pip install . + artifacts/${{ inputs.package-name }}/bin/python3 -m pip install dist/salt-${{ env.SALT_VERSION }}.tar.gz if [ ${{ inputs.platform }} == "darwin" ]; then pkg/macos/prep_salt.sh --build-dir ./artifacts/${{ inputs.package-name }} rm -rf ./artifacts/${{ inputs.package-name }}/opt @@ -82,6 +87,13 @@ runs: cd artifacts Compress-Archive -LiteralPath "${{ inputs.package-name }}" -DestinationPath "${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.zip" + - name: Upload Source Tarball as an Artifact + uses: actions/upload-artifact@v3 + with: + name: salt-${{ env.SALT_VERSION }}.tar.gz + path: dist/salt-*.tar.gz + retention-days: 7 + - name: Upload Onedir Tarball as an Artifact if: ${{ inputs.platform != 'windows' }} uses: actions/upload-artifact@v3 diff --git a/noxfile.py b/noxfile.py index 6812b153985..25caab26b58 100644 --- a/noxfile.py +++ b/noxfile.py @@ -7,11 +7,14 @@ Nox configuration script # pylint: disable=resource-leakage,3rd-party-module-not-gated import datetime +import gzip import json import os import pathlib +import shutil import sqlite3 import sys +import tarfile import tempfile # fmt: off @@ -1640,3 +1643,115 @@ def changelog(session, draft, force): # Do not ask, just remove news fragments town_cmd.append("--yes") session.run(*town_cmd) + + +class Recompress: + """ + Helper class to re-compress a ``.tag.gz`` file to make it reproducible. + """ + + def __init__(self, mtime): + self.mtime = int(mtime) + + def tar_reset(self, tarinfo): + """ + Reset user, group, mtime, and mode to create reproducible tar. + """ + tarinfo.uid = tarinfo.gid = 0 + tarinfo.uname = tarinfo.gname = "root" + tarinfo.mtime = self.mtime + if tarinfo.type == tarfile.DIRTYPE: + tarinfo.mode = 0o755 + else: + tarinfo.mode = 0o644 + if tarinfo.pax_headers: + raise ValueError(tarinfo.name, tarinfo.pax_headers) + return tarinfo + + def recompress(self, targz): + """ + Re-compress the passed path. + """ + tempd = pathlib.Path(tempfile.mkdtemp()).resolve() + d_src = tempd.joinpath("src") + d_src.mkdir() + d_tar = tempd.joinpath(targz.stem) + d_targz = tempd.joinpath(targz.name) + with tarfile.open(d_tar, "w|") as wfile: + with tarfile.open(targz, "r:gz") as rfile: + rfile.extractall(d_src) + extracted_dir = next(pathlib.Path(d_src).iterdir()) + for name in sorted(extracted_dir.rglob("*")): + wfile.add( + str(name), + filter=self.tar_reset, + recursive=False, + arcname=str(name.relative_to(d_src)), + ) + + with open(d_tar, "rb") as rfh: + with gzip.GzipFile( + fileobj=open(d_targz, "wb"), mode="wb", filename="", mtime=self.mtime + ) as gz: # pylint: disable=invalid-name + while True: + chunk = rfh.read(1024) + if not chunk: + break + gz.write(chunk) + targz.unlink() + shutil.move(str(d_targz), str(targz)) + + +@nox.session(python="3") +def build(session): + """ + Build source and binary distributions based off the current commit author date UNIX timestamp. + + The reason being, reproducible packages. + + .. code-block: shell + + git show -s --format=%at HEAD + """ + shutil.rmtree("dist/", ignore_errors=True) + if SKIP_REQUIREMENTS_INSTALL is False: + session.install( + "--progress-bar=off", + "-r", + "requirements/build.txt", + silent=PIP_INSTALL_SILENT, + ) + + timestamp = session.run( + "git", + "show", + "-s", + "--format=%at", + "HEAD", + silent=True, + log=False, + stderr=None, + ).strip() + env = {"SOURCE_DATE_EPOCH": str(timestamp)} + session.run( + "python", + "-m", + "build", + "--sdist", + str(REPO_ROOT), + env=env, + ) + # Recreate sdist to be reproducible + recompress = Recompress(timestamp) + for targz in REPO_ROOT.joinpath("dist").glob("*.tar.gz"): + session.log("Re-compressing %s...", targz.relative_to(REPO_ROOT)) + recompress.recompress(targz) + + sha256sum = shutil.which("sha256sum") + if sha256sum: + packages = [ + str(pkg.relative_to(REPO_ROOT)) + for pkg in REPO_ROOT.joinpath("dist").iterdir() + ] + session.run("sha256sum", *packages, external=True) + session.run("python", "-m", "twine", "check", "dist/*") diff --git a/requirements/build.txt b/requirements/build.txt new file mode 100644 index 00000000000..d0f72dd946a --- /dev/null +++ b/requirements/build.txt @@ -0,0 +1,2 @@ +twine +build>=0.7.0