From 453812a6ea2d6c1dc2f4ca6ca235c1081402a9b0 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 11 Apr 2023 17:16:05 +0100 Subject: [PATCH] Switch to using `tools` when uploading to S3 Signed-off-by: Pedro Algarvio --- .github/workflows/nightly.yml | 29 +++++---- .github/workflows/release.yml | 29 +++++---- .pre-commit-config.yaml | 57 +++++++++++++++-- requirements/release.in | 3 + requirements/release.txt | 69 +++++++++++++++++++++ tools/__init__.py | 1 + tools/release.py | 113 ++++++++++++++++++++++++++++++++++ tools/utils.py | 80 ++++++++++++++++++++++++ 8 files changed, 351 insertions(+), 30 deletions(-) create mode 100644 requirements/release.in create mode 100644 requirements/release.txt create mode 100644 tools/release.py create mode 100644 tools/utils.py diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 72ca6cf..4bd69c8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -56,6 +56,12 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Get Salt Project GitHub Actions Bot Environment + run: | + TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") + SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) + echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" + - name: Setup GnuPG run: | sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg @@ -86,18 +92,15 @@ jobs: rm "$SECRETS_KEY_FILE" echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install Requirements + run: | + python3 -m pip install -r requirements/release.txt + - name: Upload Develop to S3 run: | - echo "Exporting GPG Key" - gpg --output SALT-PROJECT-GPG-PUBKEY-2023.gpg --export 64CBBC8173D76B3F - echo "Exporting Armored GPG Key" - gpg --armor --output SALT-PROJECT-GPG-PUBKEY-2023.pub --export 64CBBC8173D76B3F - declare -a files=("bootstrap-salt.sh bootstrap-salt.sh.sha256 bootstrap-salt.ps1 bootstrap-salt.ps1.sha256 SALT-PROJECT-GPG-PUBKEY-2023.gpg SALT-PROJECT-GPG-PUBKEY-2023.pub") - for fname in "${files[@]}" - do - echo "GPG Signing ${fpath} ..." - gpg --local-user 64CBBC8173D76B3F --output "${fname}.asc" --armor --detach-sign --sign "${fpath}" - echo "Uploading ${fpath} and ${fpath}.asc to S3" - aws s3 cp "${fname}" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/${{ github.ref_name }}/${fname}" - aws s3 cp "${fname}.asc" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/${{ github.ref_name }}/${fname}.asc" - done + tools release s3-publish --key-id 64CBBC8173D76B3F develop diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a9c96e2..2572614 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,7 @@ jobs: - name: Install Requirements run: | - python3 -m pip install requests pre-commit + python3 -m pip install -r requirements/release.txt pre-commit install --install-hooks - name: Setup GnuPG @@ -296,9 +296,12 @@ jobs: steps: - uses: actions/checkout@v3 - with: - ref: stable - repository: ${{ github.repository }} + + - name: Get Salt Project GitHub Actions Bot Environment + run: | + TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") + SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) + echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" - name: Setup GnuPG run: | @@ -330,16 +333,18 @@ jobs: rm "$SECRETS_KEY_FILE" echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install Requirements + run: | + python3 -m pip install -r requirements/release.txt + - name: Upload Stable Release to S3 run: | - gpg --output SALT-PROJECT-GPG-PUBKEY-2023.gpg --export 64CBBC8173D76B3F - gpg --armor --output SALT-PROJECT-GPG-PUBKEY-2023.pub --export 64CBBC8173D76B3F - files="bootstrap-salt.sh bootstrap-salt.sh.sha256 bootstrap-salt.ps1 bootstrap-salt.ps1.sha256 SALT-PROJECT-GPG-PUBKEY-2023.gpg SALT-PROJECT-GPG-PUBKEY-2023.pub" - for fname in $files; do - gpg --local-user 64CBBC8173D76B3F --output "${fname}.asc" --armor --detach-sign --sign "${fpath}" - aws s3 cp "${fname}" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/stable/${fname}" - aws s3 cp "${fname}.asc" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/stable/${fname}.asc" - done + tools release s3-publish --key-id 64CBBC8173D76B3F stable update-develop-checksums: name: Update Release Checksums on Develop diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e382cca..655876b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,11 +16,6 @@ repos: hooks: - id: mdformat - - repo: https://github.com/psf/black - rev: 22.6.0 - hooks: - - id: black - - repo: https://github.com/s0undt3ch/python-tools-scripts rev: "0.12.0" hooks: @@ -34,6 +29,58 @@ repos: - pre-commit - actionlint + - repo: https://github.com/jazzband/pip-tools + rev: 6.13.0 + hooks: + - id: pip-compile + files: ^requirements/release\.(in|txt)$ + args: + - requirements/release.in + + - repo: https://github.com/asottile/pyupgrade + rev: v2.37.3 + hooks: + - id: pyupgrade + name: Rewrite Code to be Py3.9+ + args: [--py39-plus] + + - repo: https://github.com/asottile/reorder_python_imports + rev: v3.8.2 + hooks: + - id: reorder-python-imports + args: [--py39-plus] + + - repo: https://github.com/psf/black + rev: 22.6.0 + hooks: + - id: black + args: [] + +# - repo: https://github.com/PyCQA/flake8 +# rev: 3.9.2 +# hooks: +# - id: flake8 +# language_version: python3 +# additional_dependencies: +# - flake8-mypy-fork +# - flake8-docstrings +# - flake8-typing-imports +# +# - repo: https://github.com/pre-commit/mirrors-mypy +# rev: v1.0.0 +# hooks: +# - id: mypy +# alias: mypy-tools +# name: Run mypy against tools +# files: ^tools/.*\.py$ +# #args: [--strict] +# additional_dependencies: +# - attrs +# - rich +# - types-attrs +# - types-pyyaml +# - types-requests + - repo: local hooks: - id: generate-actions-workflow diff --git a/requirements/release.in b/requirements/release.in new file mode 100644 index 0000000..d0d1a73 --- /dev/null +++ b/requirements/release.in @@ -0,0 +1,3 @@ +pre-commit +python-tools-scripts >= 0.12.0 +boto3 diff --git a/requirements/release.txt b/requirements/release.txt new file mode 100644 index 0000000..b4d5a12 --- /dev/null +++ b/requirements/release.txt @@ -0,0 +1,69 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile requirements/release.in +# +attrs==22.2.0 + # via python-tools-scripts +boto3==1.26.110 + # via -r requirements/release.in +botocore==1.29.110 + # via + # boto3 + # s3transfer +certifi==2022.12.7 + # via requests +cfgv==3.3.1 + # via pre-commit +charset-normalizer==3.1.0 + # via requests +distlib==0.3.6 + # via virtualenv +filelock==3.11.0 + # via virtualenv +identify==2.5.22 + # via pre-commit +idna==3.4 + # via requests +jmespath==1.0.1 + # via + # boto3 + # botocore +markdown-it-py==2.2.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +nodeenv==1.7.0 + # via pre-commit +platformdirs==3.2.0 + # via virtualenv +pre-commit==3.2.2 + # via -r requirements/release.in +pygments==2.15.0 + # via rich +python-dateutil==2.8.2 + # via botocore +python-tools-scripts==0.12.0 + # via -r requirements/release.in +pyyaml==6.0 + # via pre-commit +requests==2.28.2 + # via python-tools-scripts +rich==13.3.3 + # via python-tools-scripts +s3transfer==0.6.0 + # via boto3 +six==1.16.0 + # via python-dateutil +typing-extensions==4.5.0 + # via python-tools-scripts +urllib3==1.26.15 + # via + # botocore + # requests +virtualenv==20.21.0 + # via pre-commit + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/tools/__init__.py b/tools/__init__.py index f1a2418..4a820ca 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -3,6 +3,7 @@ import logging import ptscripts ptscripts.register_tools_module("tools.pre_commit") +ptscripts.register_tools_module("tools.release") for name in ("boto3", "botocore", "urllib3"): logging.getLogger(name).setLevel(logging.INFO) diff --git a/tools/release.py b/tools/release.py new file mode 100644 index 0000000..2b2ee7d --- /dev/null +++ b/tools/release.py @@ -0,0 +1,113 @@ +""" +These commands are used to release Salt Bootstrap. +""" +# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated +from __future__ import annotations + +import logging +import sys +from typing import TYPE_CHECKING + +from ptscripts import command_group +from ptscripts import Context + +import tools.utils + +try: + import boto3 +except ImportError: + print( + "\nPlease run 'python -m pip install -r requirements/release.txt'\n", + file=sys.stderr, + flush=True, + ) + raise + +log = logging.getLogger(__name__) + +# Define the command group +release = command_group( + name="release", + help="Release Related Commands", + description=__doc__, +) + + +@release.command( + name="s3-publish", + arguments={ + "branch": { + "help": "The kind of publish to do.", + "choices": ("stable", "develop"), + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + }, +) +def s3_publish(ctx: Context, branch: str, key_id: str = None): + """ + Publish scripts to S3. + """ + if TYPE_CHECKING: + assert key_id + + ctx.info("Preparing upload ...") + s3 = boto3.client("s3") + + ctx.info( + f"Uploading release artifacts to {tools.utils.RELEASE_BUCKET_NAME!r} bucket ..." + ) + paths_to_upload = [ + f"{tools.utils.GPG_KEY_FILENAME}.gpg", + f"{tools.utils.GPG_KEY_FILENAME}.pub", + ] + copy_exclusions = [ + ".asc", + ".gpg", + ".pub", + ".sha256", + ] + + try: + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, tools.utils.REPO_ROOT) + + for fpath in tools.utils.REPO_ROOT.glob("bootstrap-salt.*"): + if fpath.suffix in copy_exclusions: + continue + paths_to_upload.append(fpath.name) + ret = ctx.run( + "sha256sum", + fpath.relative_to(tools.utils.REPO_ROOT), + capture=True, + check=False, + ) + if ret.returncode: + ctx.error( + f"Failed to get the sha256sum of {fpath.relative_to(tools.utils.REPO_ROOT)}" + ) + ctx.exit(1) + shasum_file = fpath.parent / f"{fpath.name}.sha256" + shasum_file.write_bytes(ret.stdout) + paths_to_upload.append(shasum_file.name) + tools.utils.gpg_sign(ctx, key_id, shasum_file) + paths_to_upload.append(f"{shasum_file.name}.asc") + tools.utils.gpg_sign(ctx, key_id, fpath) + paths_to_upload.append(f"{fpath.name}.asc") + + for path in paths_to_upload: + upload_path = f"bootstrap/{branch}/{path}" + size = fpath.stat().st_size + ctx.info(f" {upload_path}") + with tools.utils.create_progress_bar(file_progress=True) as progress: + task = progress.add_task(description="Uploading...", total=size) + s3.upload_file( + fpath, + tools.utils.RELEASE_BUCKET_NAME, + upload_path, + Callback=tools.utils.UpdateProgress(progress, task), + ) + except KeyboardInterrupt: + pass diff --git a/tools/utils.py b/tools/utils.py new file mode 100644 index 0000000..b07beb9 --- /dev/null +++ b/tools/utils.py @@ -0,0 +1,80 @@ +# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated +from __future__ import annotations + +import os +import pathlib + +from ptscripts import Context +from rich.progress import BarColumn +from rich.progress import Column +from rich.progress import DownloadColumn +from rich.progress import Progress +from rich.progress import TextColumn +from rich.progress import TimeRemainingColumn +from rich.progress import TransferSpeedColumn + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +GPG_KEY_FILENAME = "SALT-PROJECT-GPG-PUBKEY-2023" +SPB_ENVIRONMENT = os.environ.get("SPB_ENVIRONMENT") or "prod" +RELEASE_BUCKET_NAME = f"salt-project-{SPB_ENVIRONMENT}-salt-artifacts-release" + + +class UpdateProgress: + def __init__(self, progress, task): + self.progress = progress + self.task = task + + def __call__(self, chunk_size): + self.progress.update(self.task, advance=chunk_size) + + +def create_progress_bar(file_progress: bool = False, **kwargs): + if file_progress: + return Progress( + TextColumn("[progress.description]{task.description}"), + BarColumn(), + DownloadColumn(), + TransferSpeedColumn(), + TextColumn("eta"), + TimeRemainingColumn(), + **kwargs, + ) + return Progress( + TextColumn( + "[progress.description]{task.description}", table_column=Column(ratio=3) + ), + BarColumn(), + expand=True, + **kwargs, + ) + + +def export_gpg_key(ctx: Context, key_id: str, export_path: pathlib.Path): + keyfile_gpg = export_path.joinpath(GPG_KEY_FILENAME).with_suffix(".gpg") + if keyfile_gpg.exists(): + keyfile_gpg.unlink() + ctx.info(f"Exporting GnuPG Key '{key_id}' to {keyfile_gpg} ...") + ctx.run("gpg", "--output", str(keyfile_gpg), "--export", key_id) + keyfile_pub = export_path.joinpath(GPG_KEY_FILENAME).with_suffix(".pub") + if keyfile_pub.exists(): + keyfile_pub.unlink() + ctx.info(f"Exporting GnuPG Key '{key_id}' to {keyfile_pub} ...") + ctx.run("gpg", "--armor", "--output", str(keyfile_pub), "--export", key_id) + + +def gpg_sign(ctx: Context, key_id: str, path: pathlib.Path): + ctx.info(f"GPG Signing '{path}' ...") + signature_fpath = path.parent / f"{path.name}.asc" + if signature_fpath.exists(): + signature_fpath.unlink() + ctx.run( + "gpg", + "--local-user", + key_id, + "--output", + str(signature_fpath), + "--armor", + "--detach-sign", + "--sign", + str(path), + )