Start publishing to PyPi

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

.github/workflows/build-src-repo.yml

@@ -85,6 +85,17 @@ jobs:
             ${{ inputs.rc-build && '--rc-build' || ''  }} --incoming=artifacts/pkgs/incoming \
             --repo-path=artifacts/pkgs/repo
 
+      - name: Upload Standalone Repository As An Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: salt-${{ inputs.salt-version }}-${{ inputs.environment }}-src-repo
+          path: |
+            artifacts/pkgs/repo/salt/py3/src/${{ inputs.salt-version }}/salt-${{ inputs.salt-version }}.tar.gz
+            artifacts/pkgs/repo/salt/py3/src/${{ inputs.salt-version }}/salt-${{ inputs.salt-version }}.tar.gz.*
+            artifacts/pkgs/repo/salt/py3/src/${{ inputs.salt-version }}/*-GPG-*
+          retention-days: 7
+          if-no-files-found: error
+
       - name: Upload Repository As An Artifact
         uses: ./.github/actions/upload-artifact
         with:

.github/workflows/release.yml

@@ -135,6 +135,7 @@ jobs:
           EOF
 
       - name: Get Secrets
+        id: get-secrets
         env:
           SECRETS_KEY: ${{ secrets.SECRETS_KEY }}
         run: |
@@ -151,6 +152,11 @@ jobs:
           sync
           rm "$SECRETS_KEY_FILE"
           echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf"
+          TWINE_PASSWORD=$(aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/publishing/publish-pypi \
+            --query SecretString --output text | jq .default_passphrase -r | base64 -d \
+            | gpg --passphrase-file "$SECRETS_KEY_FILE" -d -)
+          echo "::add-mask::$TWINE_PASSWORD"
+          echo "twine-password=$TWINE_PASSWORD" >> "${GITHUB_OUTPUT}"
 
       - name: Configure Git
         shell: bash
@@ -197,6 +203,12 @@ jobs:
           replacesArtifacts: true
           tag: v${{ needs.prepare-workflow.outputs.salt-version }}
 
+      - name: Publish to PyPi
+        env:
+          TWINE_PASSWORD: "${{ steps.get-secrets.outputs.twine-password }}"
+        run: |
+          tools pkg pypi-upload release-artifacts/salt-${{ needs.prepare-workflow.outputs.salt-version }}.tar.gz
+
   set-pipeline-exit-status:
     # This step is just so we can make github require this step, to pass checks
     # on a pull request instead of requiring all

.github/workflows/staging.yml

@@ -903,10 +903,10 @@ jobs:
           name: salt-${{ needs.prepare-workflow.outputs.salt-version }}.patch
           path: artifacts/release
 
-      - name: Download Source Tarball
+      - name: Download Source Repository
         uses: actions/download-artifact@v3
         with:
-          name: salt-${{ needs.prepare-workflow.outputs.salt-version }}.tar.gz
+          name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-src-repo
           path: artifacts/release
 
       - name: Download Release Documentation (HTML)
@@ -925,6 +925,58 @@ jobs:
         run: |
           tools release upload-artifacts ${{ needs.prepare-workflow.outputs.salt-version }} artifacts/release
 
+  publish-pypi:
+    name: Publish to PyPi(test)
+    needs:
+      - prepare-workflow
+      - build-repositories
+    environment: staging
+    runs-on:
+      - self-hosted
+      - linux
+      - repo-staging
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Python Tools Scripts
+        uses: ./.github/actions/setup-python-tools-scripts
+
+      - name: Setup GnuPG
+        run: |
+          sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg
+          GNUPGHOME="$(mktemp -d -p /run/gpg)"
+          echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV"
+          cat <<EOF > "${GNUPGHOME}/gpg.conf"
+          batch
+          no-tty
+          pinentry-mode loopback
+          EOF
+
+      - name: Get Secrets
+        id: get-secrets
+        env:
+          SECRETS_KEY: ${{ secrets.SECRETS_KEY }}
+        run: |
+          SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX)
+          echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE"
+          TWINE_PASSWORD=$(aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/publishing/publish-test-pypi \
+            --query SecretString --output text | jq .default_passphrase -r | base64 -d \
+            | gpg --passphrase-file "$SECRETS_KEY_FILE" -d -)
+          echo "::add-mask::$TWINE_PASSWORD"
+          echo "twine-password=$TWINE_PASSWORD" >> "${GITHUB_OUTPUT}"
+
+      - name: Download Source Repository
+        uses: actions/download-artifact@v3
+        with:
+          name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-src-repo
+          path: artifacts/release
+
+      - name: Publish to Test PyPi
+        env:
+          TWINE_PASSWORD: "${{ steps.get-secrets.outputs.twine-password }}"
+        run: |
+          tools pkg pypi-upload --test artifacts/release/salt-${{ needs.prepare-workflow.outputs.salt-version }}.tar.gz
+
   set-pipeline-exit-status:
     # This step is just so we can make github require this step, to pass checks
     # on a pull request instead of requiring all
@@ -943,6 +995,7 @@ jobs:
       - build-repositories
       - publish-repositories
       - upload-release-artifacts
+      - publish-pypi
     steps:
       - name: Get workflow information
         id: get-workflow-info

.github/workflows/templates/release.yml.jinja

@@ -166,6 +166,7 @@ permissions:
           EOF
 
       - name: Get Secrets
+        id: get-secrets
         env:
           SECRETS_KEY: ${{ secrets.SECRETS_KEY }}
         run: |
@@ -182,6 +183,11 @@ permissions:
           sync
           rm "$SECRETS_KEY_FILE"
           echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf"
+          TWINE_PASSWORD=$(aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/publishing/publish-pypi \
+            --query SecretString --output text | jq .default_passphrase -r | base64 -d \
+            | gpg --passphrase-file "$SECRETS_KEY_FILE" -d -)
+          echo "::add-mask::$TWINE_PASSWORD"
+          echo "twine-password=$TWINE_PASSWORD" >> "${GITHUB_OUTPUT}"
 
       - name: Configure Git
         shell: bash
@@ -228,6 +234,10 @@ permissions:
           replacesArtifacts: true
           tag: v${{ needs.prepare-workflow.outputs.salt-version }}
 
-  {#- We need the pypi.org upload job added here #}
+      - name: Publish to PyPi
+        env:
+          TWINE_PASSWORD: "${{ steps.get-secrets.outputs.twine-password }}"
+        run: |
+          tools pkg pypi-upload release-artifacts/salt-${{ needs.prepare-workflow.outputs.salt-version }}.tar.gz
 
 <%- endblock jobs %>

.github/workflows/templates/staging.yml.jinja

@@ -83,10 +83,10 @@ concurrency:
           name: salt-${{ needs.prepare-workflow.outputs.salt-version }}.patch
           path: artifacts/release
 
-      - name: Download Source Tarball
+      - name: Download Source Repository
         uses: actions/download-artifact@v3
         with:
-          name: salt-${{ needs.prepare-workflow.outputs.salt-version }}.tar.gz
+          name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-src-repo
           path: artifacts/release
 
       - name: Download Release Documentation (HTML)
@@ -115,6 +115,59 @@ concurrency:
         run: |
           tools release upload-artifacts ${{ needs.prepare-workflow.outputs.salt-version }} artifacts/release
 
-  {#- We need the test.pypi.org upload job added here #}
   {#- We need the package upload tests against staging added here #}
+
+  publish-pypi:
+    <%- do conclusion_needs.append('publish-pypi') %>
+    name: Publish to PyPi(test)
+    needs:
+      - prepare-workflow
+      - build-repositories
+    environment: <{ gh_environment }>
+    runs-on:
+      - self-hosted
+      - linux
+      - repo-<{ gh_environment }>
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Python Tools Scripts
+        uses: ./.github/actions/setup-python-tools-scripts
+
+      - name: Setup GnuPG
+        run: |
+          sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg
+          GNUPGHOME="$(mktemp -d -p /run/gpg)"
+          echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV"
+          cat <<EOF > "${GNUPGHOME}/gpg.conf"
+          batch
+          no-tty
+          pinentry-mode loopback
+          EOF
+
+      - name: Get Secrets
+        id: get-secrets
+        env:
+          SECRETS_KEY: ${{ secrets.SECRETS_KEY }}
+        run: |
+          SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX)
+          echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE"
+          TWINE_PASSWORD=$(aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/publishing/publish-test-pypi \
+            --query SecretString --output text | jq .default_passphrase -r | base64 -d \
+            | gpg --passphrase-file "$SECRETS_KEY_FILE" -d -)
+          echo "::add-mask::$TWINE_PASSWORD"
+          echo "twine-password=$TWINE_PASSWORD" >> "${GITHUB_OUTPUT}"
+
+      - name: Download Source Repository
+        uses: actions/download-artifact@v3
+        with:
+          name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-src-repo
+          path: artifacts/release
+
+      - name: Publish to Test PyPi
+        env:
+          TWINE_PASSWORD: "${{ steps.get-secrets.outputs.twine-password }}"
+        run: |
+          tools pkg pypi-upload --test artifacts/release/salt-${{ needs.prepare-workflow.outputs.salt-version }}.tar.gz
+
 <%- endblock jobs %>

tools/pkg.py

@@ -361,3 +361,47 @@ def source_tarball(ctx: Context):
         ]
         ctx.run("sha256sum", *packages)
     ctx.run("python3", "-m", "twine", "check", "dist/*", check=True)
+
+
+@pkg.command(
+    name="pypi-upload",
+    venv_config={
+        "requirements_files": [
+            tools.utils.REPO_ROOT / "requirements" / "build.txt",
+        ]
+    },
+    arguments={
+        "files": {
+            "help": "Files to upload to PyPi",
+            "nargs": "*",
+        },
+        "test": {
+            "help": "When true, upload to test.pypi.org instead",
+        },
+    },
+)
+def pypi_upload(ctx: Context, files: list[pathlib.Path], test: bool = False):
+    ctx.run(
+        "python3", "-m", "twine", "check", *[str(fpath) for fpath in files], check=True
+    )
+    if test is True:
+        repository_url = "https://test.pypi.org/legacy/"
+    else:
+        repository_url = "https://pypi.org/legacy/"
+    if "TWINE_USERNAME" not in os.environ:
+        os.environ["TWINE_USERNAME"] = "__token__"
+    if "TWINE_PASSWORD" not in os.environ:
+        ctx.error("The 'TWINE_PASSWORD' variable is not set. Cannot upload.")
+        ctx.exit(1)
+    cmdline = [
+        "twine",
+        "upload",
+        f"--repository-url={repository_url}",
+        "--username=__token__",
+        *[str(fpath) for fpath in files],
+    ]
+    ctx.info(f"Running '{' '.join(cmdline)}' ...")
+    ret = ctx.run(*cmdline, check=False)
+    if ret.returncode:
+        ctx.error(ret.stderr.strip().decode())
+    ctx.exit(ret.returncode)

tools/pkgrepo.py

@@ -787,6 +787,8 @@ def src(
             hexdigest = _get_file_checksum(fpath, hash_name)
             with open(f"{hashes_base_path}_{hash_name.upper()}", "a+") as wfh:
                 wfh.write(f"{hexdigest} {dpath.name}\n")
+            with open(f"{dpath}.{hash_name}", "a+") as wfh:
+                wfh.write(f"{hexdigest} {dpath.name}\n")
 
     for fpath in create_repo_path.iterdir():
         tools.utils.gpg_sign(ctx, key_id, fpath)
@@ -1458,6 +1460,8 @@ def _create_onedir_based_repo(
             release_json[dpath.name][hash_name.upper()] = hexdigest
             with open(f"{hashes_base_path}_{hash_name.upper()}", "a+") as wfh:
                 wfh.write(f"{hexdigest} {dpath.name}\n")
+            with open(f"{dpath}.{hash_name}", "a+") as wfh:
+                wfh.write(f"{hexdigest} {dpath.name}\n")
 
     for fpath in create_repo_path.iterdir():
         if fpath.suffix in pkg_suffixes:

tools/pre_commit.py

@@ -60,6 +60,7 @@ def generate_workflows(ctx: Context):
             "template": "nightly.yml",
         },
         "Stage Release": {
+            "slug": "staging",
             "template": "staging.yml",
         },
         "Scheduled": {
@@ -101,6 +102,9 @@ def generate_workflows(ctx: Context):
         context = {
             "template": template_path.relative_to(tools.utils.REPO_ROOT),
             "workflow_name": workflow_name,
+            "workflow_slug": (
+                details.get("slug") or workflow_name.lower().replace(" ", "-")
+            ),
             "includes": includes,
             "conclusion_needs": NeedsTracker(),
             "test_salt_needs": NeedsTracker(),