Start building the Onedir repo

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

.github/workflows/build-onedir-repo.yml

@@ -0,0 +1,150 @@
+name: Build Onedir Repository
+
+on:
+  workflow_call:
+    inputs:
+      salt-version:
+        type: string
+        required: true
+        description: The Salt version to set prior to building packages.
+      nightly-build:
+        type: boolean
+        default: false
+      rc-build:
+        type: boolean
+        default: false
+      environment:
+        type: string
+        description: On which GitHub Environment Context To Run
+    secrets:
+      SECRETS_KEY:
+        required: true
+
+env:
+  COLUMNS: 160
+
+jobs:
+  build-repo:
+    name: Onedir
+    environment: ${{ inputs.environment }}
+    runs-on:
+      - self-hosted
+      - linux
+      - repo-${{ inputs.environment }}
+    strategy:
+      fail-fast: false
+      max-parallel: 2
+      matrix:
+        key-id:
+          - "0E08A149DE57BFBE"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Python Tools Scripts
+        uses: ./.github/actions/setup-python-tools-scripts
+
+      - name: Download Windows Packages
+        uses: actions/download-artifact@v3
+        with:
+          name: salt-${{ inputs.salt-version }}-x86-windows-pkgs
+          path: artifacts/pkgs/incoming
+
+      - name: Download Linux x86_64 Onedir Archive
+        uses: actions/download-artifact@v3
+        with:
+          name: salt-${{ inputs.salt-version }}-onedir-linux-x86_64.tar.xz
+          path: artifacts/pkgs/incoming
+
+      - name: Download Linux aarch64 Onedir Archive
+        uses: actions/download-artifact@v3
+        with:
+          name: salt-${{ inputs.salt-version }}-onedir-linux-aarch64.tar.xz
+          path: artifacts/pkgs/incoming
+
+      - name: Download macOS x86_64 Onedir Archive
+        uses: actions/download-artifact@v3
+        with:
+          name: salt-${{ inputs.salt-version }}-onedir-darwin-x86_64.tar.xz
+          path: artifacts/pkgs/incoming
+
+      - name: Download Windows amd64 Onedir Archive
+        uses: actions/download-artifact@v3
+        with:
+          name: salt-${{ inputs.salt-version }}-onedir-windows-amd64.tar.xz
+          path: artifacts/pkgs/incoming
+
+      - name: Download Windows amd64 Onedir Archive(zip)
+        uses: actions/download-artifact@v3
+        with:
+          name: salt-${{ inputs.salt-version }}-onedir-windows-amd64.zip
+          path: artifacts/pkgs/incoming
+
+      - name: Download Windows x86 Onedir Archive
+        uses: actions/download-artifact@v3
+        with:
+          name: salt-${{ inputs.salt-version }}-onedir-windows-x86.tar.xz
+          path: artifacts/pkgs/incoming
+
+      - name: Download Windows amd64 Onedir Archive(zip)
+        uses: actions/download-artifact@v3
+        with:
+          name: salt-${{ inputs.salt-version }}-onedir-windows-x86.zip
+          path: artifacts/pkgs/incoming
+
+      - name: Setup GnuPG
+        run: |
+          sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg
+          GNUPGHOME="$(mktemp -d -p /run/gpg)"
+          echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV"
+          cat <<EOF > "${GNUPGHOME}/gpg.conf"
+          batch
+          no-tty
+          pinentry-mode loopback
+          EOF
+
+      - name: Get Secrets
+        env:
+          SECRETS_KEY: ${{ secrets.SECRETS_KEY }}
+        run: |
+          SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX)
+          echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE"
+          aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys \
+            --query SecretString --output text | jq .default_key -r \
+            | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \
+            | gpg --import -
+          aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys \
+            --query SecretString --output text| jq .default_passphrase -r \
+            | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d -
+          rm "$SECRETS_KEY_FILE"
+          echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf"
+
+      - name: Create Repository Path
+        run: |
+          mkdir -p artifacts/pkgs/repo
+
+      - name: Download `salt-archive-keyring.gpg`
+        env:
+          SECRETS_KEY: ${{ secrets.SECRETS_KEY }}
+        run: |
+          SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX)
+          echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE"
+          aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/salt-archive-keyring-gpg-file \
+            --query SecretString --output text| jq .base64 -r | base64 -d \
+            | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o ~/salt-archive-keyring.gpg -d -
+          rm "${SECRETS_KEY_FILE}"
+
+      - name: Create Repository
+        run: |
+          tools pkg-repo onedir --key-id=${{ matrix.key-id }} \
+            ${{ inputs.nightly-build && '--nightly-build' || ''  }} --salt-version=${{ inputs.salt-version }} \
+            ${{ inputs.rc-build && '--rc-build' || ''  }} --incoming=artifacts/pkgs/incoming \
+            --repo-path=artifacts/pkgs/repo
+
+      - name: Upload Repository As An Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: salt-${{ inputs.salt-version }}-${{ inputs.environment }}-repo
+          path: artifacts/pkgs/repo/*
+          retention-days: 7
+          if-no-files-found: error

.github/workflows/build-repos.yml

@@ -67,3 +67,14 @@ jobs:
       rc-build: ${{ inputs.rc-build }}
     secrets:
       SECRETS_KEY: ${{ secrets.SECRETS_KEY }}
+
+  build-onedir:
+    name: Build Onedir Repositories
+    uses: ./.github/workflows/build-onedir-repo.yml
+    with:
+      environment: ${{ inputs.environment }}
+      salt-version: "${{ inputs.salt-version }}"
+      nightly-build: ${{ inputs.nightly-build }}
+      rc-build: ${{ inputs.rc-build }}
+    secrets:
+      SECRETS_KEY: ${{ secrets.SECRETS_KEY }}

tools/pkgrepo.py

@@ -751,6 +751,140 @@ def macos(
     ctx.info("Done")
 
 
+@pkg.command(
+    name="onedir",
+    arguments={
+        "salt_version": {
+            "help": "The salt version for which to build the repository",
+            "required": True,
+        },
+        "repo_path": {
+            "help": "Path where the repository shall be created.",
+            "required": True,
+        },
+        "key_id": {
+            "help": "The GnuPG key ID used to sign.",
+            "required": True,
+        },
+        "incoming": {
+            "help": (
+                "The path to the directory containing the files that should added to "
+                "the repository."
+            ),
+            "required": True,
+        },
+        "nightly_build": {
+            "help": "Developement repository target",
+        },
+        "rc_build": {
+            "help": "Release Candidate repository target",
+        },
+    },
+)
+def onedir(
+    ctx: Context,
+    salt_version: str = None,
+    incoming: pathlib.Path = None,
+    repo_path: pathlib.Path = None,
+    key_id: str = None,
+    nightly_build: bool = False,
+    rc_build: bool = False,
+):
+    """
+    Create the onedir repository.
+    """
+    if TYPE_CHECKING:
+        assert salt_version is not None
+        assert incoming is not None
+        assert repo_path is not None
+        assert key_id is not None
+    salt_archive_keyring_gpg_file = (
+        pathlib.Path("~/salt-archive-keyring.gpg").expanduser().resolve()
+    )
+    if not salt_archive_keyring_gpg_file:
+        ctx.error(f"The file '{salt_archive_keyring_gpg_file}' does not exist.")
+        ctx.exit(1)
+
+    ctx.info("Creating repository directory structure ...")
+    if nightly_build or rc_build:
+        create_repo_path = repo_path / "salt"
+    create_repo_path = create_repo_path / "py3" / "onedir"
+    repo_json_path = create_repo_path / "repo.json"
+    if nightly_build:
+        create_repo_path = create_repo_path / datetime.utcnow().strftime("%Y-%m-%d")
+    create_repo_path.mkdir(parents=True, exist_ok=True)
+
+    ctx.info("Downloading any pre-existing 'repo.json' file")
+    if nightly_build:
+        bucket_name = "salt-project-prod-salt-artifacts-nightly"
+    else:
+        bucket_name = "salt-project-prod-salt-artifacts-staging"
+
+    bucket_url = (
+        f"s3://{bucket_name}/{create_repo_path.relative_to(repo_path)}/repo.json"
+    )
+    ret = ctx.run("aws", "s3", "cp", bucket_url, create_repo_path, check=False)
+    if ret.returncode:
+        repo_json = {}
+    else:
+        repo_json = json.loads(str(repo_json_path))
+
+    if salt_version not in repo_json:
+        repo_json[salt_version] = {}
+
+    hashes_base_path = create_repo_path / f"salt-{salt_version}"
+    for fpath in incoming.iterdir():
+        if fpath.suffix not in (".xz", ".zip"):
+            ctx.info(f"Ignoring {fpath} ...")
+            continue
+        ctx.info(f"* Processing {fpath} ...")
+        dpath = create_repo_path / fpath.name
+        ctx.info(f"Copying {fpath} to {dpath} ...")
+        shutil.copyfile(fpath, dpath)
+        if "-windows-" in fpath.name:
+            distro = "windows"
+        elif "-darwin-" in fpath.name:
+            distro = "macos"
+        elif "-linux-" in fpath.name:
+            distro = "linux"
+        for arch in ("x86_64", "aarch64", "amd64", "x86"):
+            if arch in fpath.name.lower():
+                break
+        repo_json[salt_version][dpath.name] = {
+            "name": dpath.name,
+            "version": salt_version,
+            "os": distro,
+            "arch": arch,
+        }
+        for hash_name in ("blake2b", "sha512", "sha3_512"):
+            ctx.info(f"   * Calculating {hash_name} ...")
+            hexdigest = _get_file_checksum(fpath, hash_name)
+            repo_json[salt_version][dpath.name][hash_name.upper()] = hexdigest
+            with open(f"{hashes_base_path}_{hash_name.upper()}", "a+") as wfh:
+                wfh.write(f"{hexdigest} {dpath.name}\n")
+
+    for fpath in create_repo_path.iterdir():
+        if fpath.suffix in (".gpg", ".pkg"):
+            continue
+        ctx.info("GPG Signing '{fpath.relative_to(repo_path)}' ...")
+        ctx.run("gpg", "-u", key_id, "-o" f"{fpath}.asc", "-a", "-b", "-s", str(fpath))
+
+    ctx.info(f"Copying {salt_archive_keyring_gpg_file} to {create_repo_path} ...")
+    shutil.copyfile(
+        salt_archive_keyring_gpg_file,
+        create_repo_path / salt_archive_keyring_gpg_file.name,
+    )
+
+    repo_json["latest"] = repo_json[salt_version]
+    repo_json_path.write_text(json.dumps(repo_json))
+
+    ctx.info("Creating 'latest' symlink ...")
+    latest_link = create_repo_path.parent / "latest"
+    latest_link.symlink_to(create_repo_path.name)
+
+    ctx.info("Done")
+
+
 def _get_file_checksum(fpath: pathlib.Path, hash_name: str) -> str:
 
     with fpath.open("rb") as rfh: