We now also create a repository for the source tarballs.

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

.github/workflows/build-repos.yml

@@ -25,6 +25,17 @@ env:
   COLUMNS: 160
 
 jobs:
+  build-src:
+    name: Build Source Repository
+    uses: ./.github/workflows/build-src-repo.yml
+    with:
+      environment: ${{ inputs.environment }}
+      salt-version: "${{ inputs.salt-version }}"
+      nightly-build: ${{ inputs.nightly-build }}
+      rc-build: ${{ inputs.rc-build }}
+    secrets:
+      SECRETS_KEY: ${{ secrets.SECRETS_KEY }}
+
   build-deb:
     name: Build DEB Repositories
     uses: ./.github/workflows/build-deb-repo.yml

.github/workflows/build-src-repo.yml

@@ -0,0 +1,93 @@
+---
+name: Build Source Repository
+
+on:
+  workflow_call:
+    inputs:
+      salt-version:
+        type: string
+        required: true
+        description: The Salt version to set prior to building packages.
+      nightly-build:
+        type: boolean
+        default: false
+      rc-build:
+        type: boolean
+        default: false
+      environment:
+        type: string
+        description: On which GitHub Environment Context To Run
+    secrets:
+      SECRETS_KEY:
+        required: true
+
+env:
+  COLUMNS: 160
+
+jobs:
+  build-repo:
+    name: Source
+    environment: ${{ inputs.environment }}
+    runs-on:
+      - self-hosted
+      - linux
+      - repo-${{ inputs.environment }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Python Tools Scripts
+        uses: ./.github/actions/setup-python-tools-scripts
+
+      - name: Download Source Tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: salt-${{ inputs.salt-version }}.tar.gz
+          path: artifacts/pkgs/incoming
+
+      - name: Setup GnuPG
+        run: |
+          sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg
+          GNUPGHOME="$(mktemp -d -p /run/gpg)"
+          echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV"
+          cat <<EOF > "${GNUPGHOME}/gpg.conf"
+          batch
+          no-tty
+          pinentry-mode loopback
+          EOF
+
+      - name: Get Secrets
+        env:
+          SECRETS_KEY: ${{ secrets.SECRETS_KEY }}
+        run: |
+          SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX)
+          echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE"
+          aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \
+            --query SecretString --output text | jq .default_key -r | base64 -d \
+            | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \
+            | gpg --import -
+          aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \
+            --query SecretString --output text| jq .default_passphrase -r | base64 -d \
+            | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d -
+          rm "$SECRETS_KEY_FILE"
+          echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf"
+
+      - name: Create Repository Path
+        run: |
+          mkdir -p artifacts/pkgs/repo
+
+      - name: Create Repository
+        run: |
+          tools pkg repo create src --key-id=64CBBC8173D76B3F \
+            ${{ inputs.nightly-build && '--nightly-build' || ''  }} --salt-version=${{ inputs.salt-version }} \
+            ${{ inputs.rc-build && '--rc-build' || ''  }} --incoming=artifacts/pkgs/incoming \
+            --repo-path=artifacts/pkgs/repo
+
+      - name: Upload Repository As An Artifact
+        uses: ./.github/actions/upload-artifact
+        with:
+          name: salt-${{ inputs.salt-version }}-${{ inputs.environment }}-repo
+          path: artifacts/pkgs/repo/*
+          retention-days: 7
+          if-no-files-found: error
+          archive-name: src-repo

tools/pkgrepo.py

@@ -715,6 +715,84 @@ def onedir(
     ctx.info("Done")
 
 
+@create.command(
+    name="src",
+    arguments={
+        "salt_version": {
+            "help": "The salt version for which to build the repository",
+            "required": True,
+        },
+        "repo_path": {
+            "help": "Path where the repository shall be created.",
+            "required": True,
+        },
+        "key_id": {
+            "help": "The GnuPG key ID used to sign.",
+            "required": True,
+        },
+        "incoming": {
+            "help": (
+                "The path to the directory containing the files that should added to "
+                "the repository."
+            ),
+            "required": True,
+        },
+        "nightly_build": {
+            "help": "Developement repository target",
+        },
+        "rc_build": {
+            "help": "Release Candidate repository target",
+        },
+    },
+)
+def src(
+    ctx: Context,
+    salt_version: str = None,
+    incoming: pathlib.Path = None,
+    repo_path: pathlib.Path = None,
+    key_id: str = None,
+    nightly_build: bool = False,
+    rc_build: bool = False,
+):
+    """
+    Create the onedir repository.
+    """
+    if TYPE_CHECKING:
+        assert salt_version is not None
+        assert incoming is not None
+        assert repo_path is not None
+        assert key_id is not None
+
+    ctx.info("Creating repository directory structure ...")
+    create_repo_path = _create_repo_path(
+        repo_path,
+        salt_version,
+        "src",
+        rc_build=rc_build,
+        nightly_build=nightly_build,
+    )
+    hashes_base_path = create_repo_path / f"salt-{salt_version}"
+    for fpath in incoming.iterdir():
+        if fpath.suffix not in (".gz",):
+            continue
+        ctx.info(f"* Processing {fpath} ...")
+        dpath = create_repo_path / fpath.name
+        ctx.info(f"Copying {fpath} to {dpath} ...")
+        shutil.copyfile(fpath, dpath)
+        for hash_name in ("blake2b", "sha512", "sha3_512"):
+            ctx.info(f"   * Calculating {hash_name} ...")
+            hexdigest = _get_file_checksum(fpath, hash_name)
+            with open(f"{hashes_base_path}_{hash_name.upper()}", "a+") as wfh:
+                wfh.write(f"{hexdigest} {dpath.name}\n")
+
+    for fpath in create_repo_path.iterdir():
+        tools.utils.gpg_sign(ctx, key_id, fpath)
+
+    # Export the GPG key in use
+    tools.utils.export_gpg_key(ctx, key_id, repo_path, create_repo_path)
+    ctx.info("Done")
+
+
 @publish.command(
     arguments={
         "repo_path": {
@@ -1113,7 +1191,9 @@ def _create_repo_path(
     if distro_arch:
         create_repo_path = create_repo_path / distro_arch
     if nightly_build is False:
-        create_repo_path = create_repo_path / "minor" / salt_version
+        if distro != "src":
+            create_repo_path = create_repo_path / "minor"
+        create_repo_path = create_repo_path / salt_version
     else:
         create_repo_path = create_repo_path / datetime.utcnow().strftime("%Y-%m-%d")
     create_repo_path.mkdir(exist_ok=True, parents=True)