Switch to using tools when uploading to S3

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

.github/workflows/nightly.yml

@@ -56,6 +56,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Get Salt Project GitHub Actions Bot Environment
+        run: |
+          TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
+          SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
+          echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"
+
       - name: Setup GnuPG
         run: |
           sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg
@@ -86,18 +92,15 @@ jobs:
           rm "$SECRETS_KEY_FILE"
           echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf"
 
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install Requirements
+        run: |
+          python3 -m pip install -r requirements/release.txt
+
       - name: Upload Develop to S3
         run: |
-          echo "Exporting GPG Key"
-          gpg --output SALT-PROJECT-GPG-PUBKEY-2023.gpg --export 64CBBC8173D76B3F
-          echo "Exporting Armored GPG Key"
-          gpg --armor --output SALT-PROJECT-GPG-PUBKEY-2023.pub --export 64CBBC8173D76B3F
-          declare -a files=("bootstrap-salt.sh bootstrap-salt.sh.sha256 bootstrap-salt.ps1 bootstrap-salt.ps1.sha256 SALT-PROJECT-GPG-PUBKEY-2023.gpg SALT-PROJECT-GPG-PUBKEY-2023.pub")
-          for fname in "${files[@]}"
-          do
-            echo "GPG Signing ${fpath} ..."
-            gpg --local-user 64CBBC8173D76B3F --output "${fname}.asc" --armor --detach-sign --sign "${fpath}"
-            echo "Uploading ${fpath} and ${fpath}.asc to S3"
-            aws s3 cp "${fname}" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/${{ github.ref_name }}/${fname}"
-            aws s3 cp "${fname}.asc" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/${{ github.ref_name }}/${fname}.asc"
-          done
+          tools release s3-publish --key-id 64CBBC8173D76B3F develop

.github/workflows/release.yml

@@ -65,7 +65,7 @@ jobs:
 
       - name: Install Requirements
         run: |
-          python3 -m pip install requests pre-commit
+          python3 -m pip install -r requirements/release.txt
           pre-commit install --install-hooks
 
       - name: Setup GnuPG
@@ -296,9 +296,12 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-        with:
-          ref: stable
-          repository: ${{ github.repository }}
+
+      - name: Get Salt Project GitHub Actions Bot Environment
+        run: |
+          TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
+          SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
+          echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"
 
       - name: Setup GnuPG
         run: |
@@ -330,16 +333,18 @@ jobs:
           rm "$SECRETS_KEY_FILE"
           echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf"
 
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install Requirements
+        run: |
+          python3 -m pip install -r requirements/release.txt
+
       - name: Upload Stable Release to S3
         run: |
-          gpg --output SALT-PROJECT-GPG-PUBKEY-2023.gpg --export 64CBBC8173D76B3F
-          gpg --armor --output SALT-PROJECT-GPG-PUBKEY-2023.pub --export 64CBBC8173D76B3F
-          files="bootstrap-salt.sh bootstrap-salt.sh.sha256 bootstrap-salt.ps1 bootstrap-salt.ps1.sha256 SALT-PROJECT-GPG-PUBKEY-2023.gpg SALT-PROJECT-GPG-PUBKEY-2023.pub"
-          for fname in $files; do
-            gpg --local-user 64CBBC8173D76B3F --output "${fname}.asc" --armor --detach-sign --sign "${fpath}"
-            aws s3 cp "${fname}" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/stable/${fname}"
-            aws s3 cp "${fname}.asc" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/stable/${fname}.asc"
-          done
+          tools release s3-publish --key-id 64CBBC8173D76B3F stable
 
   update-develop-checksums:
     name: Update Release Checksums on Develop

.pre-commit-config.yaml

@@ -16,11 +16,6 @@ repos:
     hooks:
       - id: mdformat
 
-  - repo: https://github.com/psf/black
-    rev: 22.6.0
-    hooks:
-      - id: black
-
   - repo: https://github.com/s0undt3ch/python-tools-scripts
     rev: "0.12.0"
     hooks:
@@ -34,6 +29,58 @@ repos:
           - pre-commit
           - actionlint
 
+  - repo: https://github.com/jazzband/pip-tools
+    rev: 6.13.0
+    hooks:
+      - id: pip-compile
+        files: ^requirements/release\.(in|txt)$
+        args:
+          - requirements/release.in
+
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v2.37.3
+    hooks:
+      - id: pyupgrade
+        name: Rewrite Code to be Py3.9+
+        args: [--py39-plus]
+
+  - repo: https://github.com/asottile/reorder_python_imports
+    rev: v3.8.2
+    hooks:
+      - id: reorder-python-imports
+        args: [--py39-plus]
+
+  - repo: https://github.com/psf/black
+    rev: 22.6.0
+    hooks:
+      - id: black
+        args: []
+
+#  - repo: https://github.com/PyCQA/flake8
+#    rev: 3.9.2
+#    hooks:
+#      - id: flake8
+#        language_version: python3
+#        additional_dependencies:
+#          - flake8-mypy-fork
+#          - flake8-docstrings
+#          - flake8-typing-imports
+#
+#  - repo: https://github.com/pre-commit/mirrors-mypy
+#    rev: v1.0.0
+#    hooks:
+#      - id: mypy
+#        alias: mypy-tools
+#        name: Run mypy against tools
+#        files: ^tools/.*\.py$
+#        #args: [--strict]
+#        additional_dependencies:
+#          - attrs
+#          - rich
+#          - types-attrs
+#          - types-pyyaml
+#          - types-requests
+
   - repo: local
     hooks:
       - id: generate-actions-workflow

requirements/release.in

@@ -0,0 +1,3 @@
+pre-commit
+python-tools-scripts >= 0.12.0
+boto3

requirements/release.txt

@@ -0,0 +1,69 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    pip-compile requirements/release.in
+#
+attrs==22.2.0
+    # via python-tools-scripts
+boto3==1.26.110
+    # via -r requirements/release.in
+botocore==1.29.110
+    # via
+    #   boto3
+    #   s3transfer
+certifi==2022.12.7
+    # via requests
+cfgv==3.3.1
+    # via pre-commit
+charset-normalizer==3.1.0
+    # via requests
+distlib==0.3.6
+    # via virtualenv
+filelock==3.11.0
+    # via virtualenv
+identify==2.5.22
+    # via pre-commit
+idna==3.4
+    # via requests
+jmespath==1.0.1
+    # via
+    #   boto3
+    #   botocore
+markdown-it-py==2.2.0
+    # via rich
+mdurl==0.1.2
+    # via markdown-it-py
+nodeenv==1.7.0
+    # via pre-commit
+platformdirs==3.2.0
+    # via virtualenv
+pre-commit==3.2.2
+    # via -r requirements/release.in
+pygments==2.15.0
+    # via rich
+python-dateutil==2.8.2
+    # via botocore
+python-tools-scripts==0.12.0
+    # via -r requirements/release.in
+pyyaml==6.0
+    # via pre-commit
+requests==2.28.2
+    # via python-tools-scripts
+rich==13.3.3
+    # via python-tools-scripts
+s3transfer==0.6.0
+    # via boto3
+six==1.16.0
+    # via python-dateutil
+typing-extensions==4.5.0
+    # via python-tools-scripts
+urllib3==1.26.15
+    # via
+    #   botocore
+    #   requests
+virtualenv==20.21.0
+    # via pre-commit
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

tools/__init__.py

@@ -3,6 +3,7 @@ import logging
 import ptscripts
 
 ptscripts.register_tools_module("tools.pre_commit")
+ptscripts.register_tools_module("tools.release")
 
 for name in ("boto3", "botocore", "urllib3"):
     logging.getLogger(name).setLevel(logging.INFO)

tools/release.py

@@ -0,0 +1,113 @@
+"""
+These commands are used to release Salt Bootstrap.
+"""
+# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated
+from __future__ import annotations
+
+import logging
+import sys
+from typing import TYPE_CHECKING
+
+from ptscripts import command_group
+from ptscripts import Context
+
+import tools.utils
+
+try:
+    import boto3
+except ImportError:
+    print(
+        "\nPlease run 'python -m pip install -r requirements/release.txt'\n",
+        file=sys.stderr,
+        flush=True,
+    )
+    raise
+
+log = logging.getLogger(__name__)
+
+# Define the command group
+release = command_group(
+    name="release",
+    help="Release Related Commands",
+    description=__doc__,
+)
+
+
+@release.command(
+    name="s3-publish",
+    arguments={
+        "branch": {
+            "help": "The kind of publish to do.",
+            "choices": ("stable", "develop"),
+        },
+        "key_id": {
+            "help": "The GnuPG key ID used to sign.",
+            "required": True,
+        },
+    },
+)
+def s3_publish(ctx: Context, branch: str, key_id: str = None):
+    """
+    Publish scripts to S3.
+    """
+    if TYPE_CHECKING:
+        assert key_id
+
+    ctx.info("Preparing upload ...")
+    s3 = boto3.client("s3")
+
+    ctx.info(
+        f"Uploading release artifacts to {tools.utils.RELEASE_BUCKET_NAME!r} bucket ..."
+    )
+    paths_to_upload = [
+        f"{tools.utils.GPG_KEY_FILENAME}.gpg",
+        f"{tools.utils.GPG_KEY_FILENAME}.pub",
+    ]
+    copy_exclusions = [
+        ".asc",
+        ".gpg",
+        ".pub",
+        ".sha256",
+    ]
+
+    try:
+        # Export the GPG key in use
+        tools.utils.export_gpg_key(ctx, key_id, tools.utils.REPO_ROOT)
+
+        for fpath in tools.utils.REPO_ROOT.glob("bootstrap-salt.*"):
+            if fpath.suffix in copy_exclusions:
+                continue
+            paths_to_upload.append(fpath.name)
+            ret = ctx.run(
+                "sha256sum",
+                fpath.relative_to(tools.utils.REPO_ROOT),
+                capture=True,
+                check=False,
+            )
+            if ret.returncode:
+                ctx.error(
+                    f"Failed to get the sha256sum of {fpath.relative_to(tools.utils.REPO_ROOT)}"
+                )
+                ctx.exit(1)
+            shasum_file = fpath.parent / f"{fpath.name}.sha256"
+            shasum_file.write_bytes(ret.stdout)
+            paths_to_upload.append(shasum_file.name)
+            tools.utils.gpg_sign(ctx, key_id, shasum_file)
+            paths_to_upload.append(f"{shasum_file.name}.asc")
+            tools.utils.gpg_sign(ctx, key_id, fpath)
+            paths_to_upload.append(f"{fpath.name}.asc")
+
+        for path in paths_to_upload:
+            upload_path = f"bootstrap/{branch}/{path}"
+            size = fpath.stat().st_size
+            ctx.info(f"  {upload_path}")
+            with tools.utils.create_progress_bar(file_progress=True) as progress:
+                task = progress.add_task(description="Uploading...", total=size)
+                s3.upload_file(
+                    fpath,
+                    tools.utils.RELEASE_BUCKET_NAME,
+                    upload_path,
+                    Callback=tools.utils.UpdateProgress(progress, task),
+                )
+    except KeyboardInterrupt:
+        pass

tools/utils.py

@@ -0,0 +1,80 @@
+# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated
+from __future__ import annotations
+
+import os
+import pathlib
+
+from ptscripts import Context
+from rich.progress import BarColumn
+from rich.progress import Column
+from rich.progress import DownloadColumn
+from rich.progress import Progress
+from rich.progress import TextColumn
+from rich.progress import TimeRemainingColumn
+from rich.progress import TransferSpeedColumn
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent
+GPG_KEY_FILENAME = "SALT-PROJECT-GPG-PUBKEY-2023"
+SPB_ENVIRONMENT = os.environ.get("SPB_ENVIRONMENT") or "prod"
+RELEASE_BUCKET_NAME = f"salt-project-{SPB_ENVIRONMENT}-salt-artifacts-release"
+
+
+class UpdateProgress:
+    def __init__(self, progress, task):
+        self.progress = progress
+        self.task = task
+
+    def __call__(self, chunk_size):
+        self.progress.update(self.task, advance=chunk_size)
+
+
+def create_progress_bar(file_progress: bool = False, **kwargs):
+    if file_progress:
+        return Progress(
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            DownloadColumn(),
+            TransferSpeedColumn(),
+            TextColumn("eta"),
+            TimeRemainingColumn(),
+            **kwargs,
+        )
+    return Progress(
+        TextColumn(
+            "[progress.description]{task.description}", table_column=Column(ratio=3)
+        ),
+        BarColumn(),
+        expand=True,
+        **kwargs,
+    )
+
+
+def export_gpg_key(ctx: Context, key_id: str, export_path: pathlib.Path):
+    keyfile_gpg = export_path.joinpath(GPG_KEY_FILENAME).with_suffix(".gpg")
+    if keyfile_gpg.exists():
+        keyfile_gpg.unlink()
+    ctx.info(f"Exporting GnuPG Key '{key_id}' to {keyfile_gpg} ...")
+    ctx.run("gpg", "--output", str(keyfile_gpg), "--export", key_id)
+    keyfile_pub = export_path.joinpath(GPG_KEY_FILENAME).with_suffix(".pub")
+    if keyfile_pub.exists():
+        keyfile_pub.unlink()
+    ctx.info(f"Exporting GnuPG Key '{key_id}' to {keyfile_pub} ...")
+    ctx.run("gpg", "--armor", "--output", str(keyfile_pub), "--export", key_id)
+
+
+def gpg_sign(ctx: Context, key_id: str, path: pathlib.Path):
+    ctx.info(f"GPG Signing '{path}' ...")
+    signature_fpath = path.parent / f"{path.name}.asc"
+    if signature_fpath.exists():
+        signature_fpath.unlink()
+    ctx.run(
+        "gpg",
+        "--local-user",
+        key_id,
+        "--output",
+        str(signature_fpath),
+        "--armor",
+        "--detach-sign",
+        "--sign",
+        str(path),
+    )