diff --git a/.github/workflows/build-src-repo.yml b/.github/workflows/build-src-repo.yml index 6f31c6a6489..6766b83c54f 100644 --- a/.github/workflows/build-src-repo.yml +++ b/.github/workflows/build-src-repo.yml @@ -85,6 +85,17 @@ jobs: ${{ inputs.rc-build && '--rc-build' || '' }} --incoming=artifacts/pkgs/incoming \ --repo-path=artifacts/pkgs/repo + - name: Upload Standalone Repository As An Artifact + uses: actions/upload-artifact@v3 + with: + name: salt-${{ inputs.salt-version }}-${{ inputs.environment }}-src-repo + path: | + artifacts/pkgs/repo/salt/py3/src/${{ inputs.salt-version }}/salt-${{ inputs.salt-version }}.tar.gz + artifacts/pkgs/repo/salt/py3/src/${{ inputs.salt-version }}/salt-${{ inputs.salt-version }}.tar.gz.* + artifacts/pkgs/repo/salt/py3/src/${{ inputs.salt-version }}/*-GPG-* + retention-days: 7 + if-no-files-found: error + - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 43936af0b40..4e18d742f13 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -135,6 +135,7 @@ jobs: EOF - name: Get Secrets + id: get-secrets env: SECRETS_KEY: ${{ secrets.SECRETS_KEY }} run: | @@ -151,6 +152,11 @@ jobs: sync rm "$SECRETS_KEY_FILE" echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + TWINE_PASSWORD=$(aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/publishing/publish-pypi \ + --query SecretString --output text | jq .default_passphrase -r | base64 -d \ + | gpg --passphrase-file "$SECRETS_KEY_FILE" -d -) + echo "::add-mask::$TWINE_PASSWORD" + echo "twine-password=$TWINE_PASSWORD" >> "${GITHUB_OUTPUT}" - name: Configure Git shell: bash @@ -197,6 +203,12 @@ jobs: replacesArtifacts: true tag: v${{ needs.prepare-workflow.outputs.salt-version }} + - name: Publish to PyPi + env: + TWINE_PASSWORD: "${{ steps.get-secrets.outputs.twine-password }}" + run: | + tools pkg pypi-upload release-artifacts/salt-${{ needs.prepare-workflow.outputs.salt-version }}.tar.gz + set-pipeline-exit-status: # This step is just so we can make github require this step, to pass checks # on a pull request instead of requiring all diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 589e161836a..0acfa3aa4ba 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -903,10 +903,10 @@ jobs: name: salt-${{ needs.prepare-workflow.outputs.salt-version }}.patch path: artifacts/release - - name: Download Source Tarball + - name: Download Source Repository uses: actions/download-artifact@v3 with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}.tar.gz + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-src-repo path: artifacts/release - name: Download Release Documentation (HTML) @@ -925,6 +925,58 @@ jobs: run: | tools release upload-artifacts ${{ needs.prepare-workflow.outputs.salt-version }} artifacts/release + publish-pypi: + name: Publish to PyPi(test) + needs: + - prepare-workflow + - build-repositories + environment: staging + runs-on: + - self-hosted + - linux + - repo-staging + steps: + - uses: actions/checkout@v3 + + - name: Setup Python Tools Scripts + uses: ./.github/actions/setup-python-tools-scripts + + - name: Setup GnuPG + run: | + sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + GNUPGHOME="$(mktemp -d -p /run/gpg)" + echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + cat < "${GNUPGHOME}/gpg.conf" + batch + no-tty + pinentry-mode loopback + EOF + + - name: Get Secrets + id: get-secrets + env: + SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + run: | + SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + TWINE_PASSWORD=$(aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/publishing/publish-test-pypi \ + --query SecretString --output text | jq .default_passphrase -r | base64 -d \ + | gpg --passphrase-file "$SECRETS_KEY_FILE" -d -) + echo "::add-mask::$TWINE_PASSWORD" + echo "twine-password=$TWINE_PASSWORD" >> "${GITHUB_OUTPUT}" + + - name: Download Source Repository + uses: actions/download-artifact@v3 + with: + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-src-repo + path: artifacts/release + + - name: Publish to Test PyPi + env: + TWINE_PASSWORD: "${{ steps.get-secrets.outputs.twine-password }}" + run: | + tools pkg pypi-upload --test artifacts/release/salt-${{ needs.prepare-workflow.outputs.salt-version }}.tar.gz + set-pipeline-exit-status: # This step is just so we can make github require this step, to pass checks # on a pull request instead of requiring all @@ -943,6 +995,7 @@ jobs: - build-repositories - publish-repositories - upload-release-artifacts + - publish-pypi steps: - name: Get workflow information id: get-workflow-info diff --git a/.github/workflows/templates/release.yml.jinja b/.github/workflows/templates/release.yml.jinja index fa3c57e35b0..1405919a793 100644 --- a/.github/workflows/templates/release.yml.jinja +++ b/.github/workflows/templates/release.yml.jinja @@ -166,6 +166,7 @@ permissions: EOF - name: Get Secrets + id: get-secrets env: SECRETS_KEY: ${{ secrets.SECRETS_KEY }} run: | @@ -182,6 +183,11 @@ permissions: sync rm "$SECRETS_KEY_FILE" echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + TWINE_PASSWORD=$(aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/publishing/publish-pypi \ + --query SecretString --output text | jq .default_passphrase -r | base64 -d \ + | gpg --passphrase-file "$SECRETS_KEY_FILE" -d -) + echo "::add-mask::$TWINE_PASSWORD" + echo "twine-password=$TWINE_PASSWORD" >> "${GITHUB_OUTPUT}" - name: Configure Git shell: bash @@ -228,6 +234,10 @@ permissions: replacesArtifacts: true tag: v${{ needs.prepare-workflow.outputs.salt-version }} - {#- We need the pypi.org upload job added here #} + - name: Publish to PyPi + env: + TWINE_PASSWORD: "${{ steps.get-secrets.outputs.twine-password }}" + run: | + tools pkg pypi-upload release-artifacts/salt-${{ needs.prepare-workflow.outputs.salt-version }}.tar.gz <%- endblock jobs %> diff --git a/.github/workflows/templates/staging.yml.jinja b/.github/workflows/templates/staging.yml.jinja index 455a4e565fe..ae46bdf508d 100644 --- a/.github/workflows/templates/staging.yml.jinja +++ b/.github/workflows/templates/staging.yml.jinja @@ -83,10 +83,10 @@ concurrency: name: salt-${{ needs.prepare-workflow.outputs.salt-version }}.patch path: artifacts/release - - name: Download Source Tarball + - name: Download Source Repository uses: actions/download-artifact@v3 with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}.tar.gz + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-src-repo path: artifacts/release - name: Download Release Documentation (HTML) @@ -115,6 +115,59 @@ concurrency: run: | tools release upload-artifacts ${{ needs.prepare-workflow.outputs.salt-version }} artifacts/release - {#- We need the test.pypi.org upload job added here #} {#- We need the package upload tests against staging added here #} + + publish-pypi: + <%- do conclusion_needs.append('publish-pypi') %> + name: Publish to PyPi(test) + needs: + - prepare-workflow + - build-repositories + environment: <{ gh_environment }> + runs-on: + - self-hosted + - linux + - repo-<{ gh_environment }> + steps: + - uses: actions/checkout@v3 + + - name: Setup Python Tools Scripts + uses: ./.github/actions/setup-python-tools-scripts + + - name: Setup GnuPG + run: | + sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + GNUPGHOME="$(mktemp -d -p /run/gpg)" + echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + cat < "${GNUPGHOME}/gpg.conf" + batch + no-tty + pinentry-mode loopback + EOF + + - name: Get Secrets + id: get-secrets + env: + SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + run: | + SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + TWINE_PASSWORD=$(aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/publishing/publish-test-pypi \ + --query SecretString --output text | jq .default_passphrase -r | base64 -d \ + | gpg --passphrase-file "$SECRETS_KEY_FILE" -d -) + echo "::add-mask::$TWINE_PASSWORD" + echo "twine-password=$TWINE_PASSWORD" >> "${GITHUB_OUTPUT}" + + - name: Download Source Repository + uses: actions/download-artifact@v3 + with: + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-src-repo + path: artifacts/release + + - name: Publish to Test PyPi + env: + TWINE_PASSWORD: "${{ steps.get-secrets.outputs.twine-password }}" + run: | + tools pkg pypi-upload --test artifacts/release/salt-${{ needs.prepare-workflow.outputs.salt-version }}.tar.gz + <%- endblock jobs %> diff --git a/tools/pkg.py b/tools/pkg.py index 7ff2f286849..5820cdab332 100644 --- a/tools/pkg.py +++ b/tools/pkg.py @@ -361,3 +361,47 @@ def source_tarball(ctx: Context): ] ctx.run("sha256sum", *packages) ctx.run("python3", "-m", "twine", "check", "dist/*", check=True) + + +@pkg.command( + name="pypi-upload", + venv_config={ + "requirements_files": [ + tools.utils.REPO_ROOT / "requirements" / "build.txt", + ] + }, + arguments={ + "files": { + "help": "Files to upload to PyPi", + "nargs": "*", + }, + "test": { + "help": "When true, upload to test.pypi.org instead", + }, + }, +) +def pypi_upload(ctx: Context, files: list[pathlib.Path], test: bool = False): + ctx.run( + "python3", "-m", "twine", "check", *[str(fpath) for fpath in files], check=True + ) + if test is True: + repository_url = "https://test.pypi.org/legacy/" + else: + repository_url = "https://pypi.org/legacy/" + if "TWINE_USERNAME" not in os.environ: + os.environ["TWINE_USERNAME"] = "__token__" + if "TWINE_PASSWORD" not in os.environ: + ctx.error("The 'TWINE_PASSWORD' variable is not set. Cannot upload.") + ctx.exit(1) + cmdline = [ + "twine", + "upload", + f"--repository-url={repository_url}", + "--username=__token__", + *[str(fpath) for fpath in files], + ] + ctx.info(f"Running '{' '.join(cmdline)}' ...") + ret = ctx.run(*cmdline, check=False) + if ret.returncode: + ctx.error(ret.stderr.strip().decode()) + ctx.exit(ret.returncode) diff --git a/tools/pkgrepo.py b/tools/pkgrepo.py index b066bef4cc3..1bcbbde1fbd 100644 --- a/tools/pkgrepo.py +++ b/tools/pkgrepo.py @@ -787,6 +787,8 @@ def src( hexdigest = _get_file_checksum(fpath, hash_name) with open(f"{hashes_base_path}_{hash_name.upper()}", "a+") as wfh: wfh.write(f"{hexdigest} {dpath.name}\n") + with open(f"{dpath}.{hash_name}", "a+") as wfh: + wfh.write(f"{hexdigest} {dpath.name}\n") for fpath in create_repo_path.iterdir(): tools.utils.gpg_sign(ctx, key_id, fpath) @@ -1458,6 +1460,8 @@ def _create_onedir_based_repo( release_json[dpath.name][hash_name.upper()] = hexdigest with open(f"{hashes_base_path}_{hash_name.upper()}", "a+") as wfh: wfh.write(f"{hexdigest} {dpath.name}\n") + with open(f"{dpath}.{hash_name}", "a+") as wfh: + wfh.write(f"{hexdigest} {dpath.name}\n") for fpath in create_repo_path.iterdir(): if fpath.suffix in pkg_suffixes: diff --git a/tools/pre_commit.py b/tools/pre_commit.py index 84c9b84eb10..396ca295cc6 100644 --- a/tools/pre_commit.py +++ b/tools/pre_commit.py @@ -60,6 +60,7 @@ def generate_workflows(ctx: Context): "template": "nightly.yml", }, "Stage Release": { + "slug": "staging", "template": "staging.yml", }, "Scheduled": { @@ -101,6 +102,9 @@ def generate_workflows(ctx: Context): context = { "template": template_path.relative_to(tools.utils.REPO_ROOT), "workflow_name": workflow_name, + "workflow_slug": ( + details.get("slug") or workflow_name.lower().replace(" ", "-") + ), "includes": includes, "conclusion_needs": NeedsTracker(), "test_salt_needs": NeedsTracker(),