Better way of restarting failed jobs

This commit is contained in:
Daniel A. Wozniak 2025-02-03 14:39:45 -07:00 committed by Daniel Wozniak
parent 98dd3e89ba
commit b3ae80492e
8 changed files with 48 additions and 508 deletions

View file

@ -2141,108 +2141,3 @@ jobs:
path: artifacts/coverage/html/full
retention-days: 7
if-no-files-found: error
set-pipeline-exit-status:
# This step is just so we can make github require this step, to pass checks
# on a pull request instead of requiring all
name: Set the ${{ github.workflow }} Pipeline Exit Status
if: always()
runs-on: ubuntu-latest
needs:
- prepare-workflow
- pre-commit
- lint
- build-docs
- build-deps-onedir
- build-salt-onedir
- build-pkgs-src
- combine-all-code-coverage
- build-ci-deps
- windows-2016
- windows-2019
- windows-2022
- macos-12
- macos-13
- macos-13-arm64
- rockylinux-8
- rockylinux-8-arm64
- rockylinux-9
- rockylinux-9-arm64
- amazonlinux-2
- amazonlinux-2-arm64
- amazonlinux-2023
- amazonlinux-2023-arm64
- archlinux-lts
- debian-11
- debian-11-arm64
- debian-12
- debian-12-arm64
- fedora-40
- opensuse-15
- photonos-4
- photonos-4-arm64
- photonos-4-fips
- photonos-4-arm64-fips
- photonos-5
- photonos-5-arm64
- photonos-5-fips
- photonos-5-arm64-fips
- ubuntu-2004
- ubuntu-2004-arm64
- ubuntu-2204
- ubuntu-2204-arm64
- ubuntu-2404
- ubuntu-2404-arm64
- rockylinux-8-pkg-tests
- rockylinux-8-arm64-pkg-tests
- rockylinux-9-pkg-tests
- rockylinux-9-arm64-pkg-tests
- amazonlinux-2-pkg-tests
- amazonlinux-2-arm64-pkg-tests
- amazonlinux-2023-pkg-tests
- amazonlinux-2023-arm64-pkg-tests
- debian-11-pkg-tests
- debian-11-arm64-pkg-tests
- debian-12-pkg-tests
- debian-12-arm64-pkg-tests
- photonos-4-pkg-tests
- photonos-4-arm64-pkg-tests
- photonos-4-pkg-tests-fips
- photonos-4-arm64-pkg-tests-fips
- photonos-5-pkg-tests
- photonos-5-arm64-pkg-tests
- photonos-5-pkg-tests-fips
- photonos-5-arm64-pkg-tests-fips
- ubuntu-2004-pkg-tests
- ubuntu-2004-arm64-pkg-tests
- ubuntu-2204-pkg-tests
- ubuntu-2204-arm64-pkg-tests
- ubuntu-2404-pkg-tests
- ubuntu-2404-arm64-pkg-tests
- macos-12-pkg-tests
- macos-13-pkg-tests
- macos-13-arm64-pkg-tests
- windows-2016-nsis-pkg-tests
- windows-2016-msi-pkg-tests
- windows-2019-nsis-pkg-tests
- windows-2019-msi-pkg-tests
- windows-2022-nsis-pkg-tests
- windows-2022-msi-pkg-tests
steps:
- name: Get workflow information
id: get-workflow-info
uses: im-open/workflow-conclusion@v2
- name: Set Pipeline Exit Status
shell: bash
run: |
if [ "${{ steps.get-workflow-info.outputs.workflow_conclusion }}" != "success" ]; then
exit 1
else
exit 0
fi
- name: Done
if: always()
run:
echo "All worflows finished"

View file

@ -3037,135 +3037,3 @@ jobs:
SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }}
run: |
tools pkg repo publish nightly --salt-version=${{ needs.prepare-workflow.outputs.salt-version }} artifacts/pkgs/repo/
set-pipeline-exit-status:
# This step is just so we can make github require this step, to pass checks
# on a pull request instead of requiring all
name: Set the ${{ github.workflow }} Pipeline Exit Status
if: always()
runs-on: ubuntu-latest
environment: nightly
needs:
- workflow-requirements
- trigger-branch-nightly-builds
- prepare-workflow
- pre-commit
- lint
- build-docs
- build-deps-onedir
- build-salt-onedir
- build-pkgs-src
- combine-all-code-coverage
- publish-repositories
- rockylinux-8-pkg-tests
- rockylinux-8-arm64-pkg-tests
- rockylinux-9-pkg-tests
- rockylinux-9-arm64-pkg-tests
- amazonlinux-2-pkg-tests
- amazonlinux-2-arm64-pkg-tests
- amazonlinux-2023-pkg-tests
- amazonlinux-2023-arm64-pkg-tests
- debian-11-pkg-tests
- debian-11-arm64-pkg-tests
- debian-12-pkg-tests
- debian-12-arm64-pkg-tests
- photonos-4-pkg-tests
- photonos-4-arm64-pkg-tests
- photonos-4-pkg-tests-fips
- photonos-4-arm64-pkg-tests-fips
- photonos-5-pkg-tests
- photonos-5-arm64-pkg-tests
- photonos-5-pkg-tests-fips
- photonos-5-arm64-pkg-tests-fips
- ubuntu-2004-pkg-tests
- ubuntu-2004-arm64-pkg-tests
- ubuntu-2204-pkg-tests
- ubuntu-2204-arm64-pkg-tests
- ubuntu-2404-pkg-tests
- ubuntu-2404-arm64-pkg-tests
- macos-12-pkg-tests
- macos-13-pkg-tests
- macos-13-arm64-pkg-tests
- windows-2016-nsis-pkg-tests
- windows-2016-msi-pkg-tests
- windows-2019-nsis-pkg-tests
- windows-2019-msi-pkg-tests
- windows-2022-nsis-pkg-tests
- windows-2022-msi-pkg-tests
steps:
- name: Get workflow information
id: get-workflow-info
uses: im-open/workflow-conclusion@v2
- name: Notify Slack
id: slack
if: always()
uses: slackapi/slack-github-action@v1.24.0
with:
payload: |
{
"attachments": [
{
"fallback": "${{ github.workflow }} Workflow build result for the `${{ github.ref_name }}` branch(attempt: ${{ github.run_attempt }}): `${{ steps.get-workflow-info.outputs.conclusion }}`\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
"color": "${{ steps.get-workflow-info.outputs.conclusion != 'success' && 'ff3d00' || '00e676' }}",
"fields": [
{
"title": "Workflow",
"short": true,
"value": "${{ github.workflow }}",
"type": "mrkdwn"
},
{
"title": "Workflow Run",
"short": true,
"value": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.run_id }}>",
"type": "mrkdwn"
},
{
"title": "Branch",
"short": true,
"value": "${{ github.ref_name }}",
"type": "mrkdwn"
},
{
"title": "Commit",
"short": true,
"value": "<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>",
"type": "mrkdwn"
},
{
"title": "Attempt",
"short": true,
"value": "${{ github.run_attempt }}",
"type": "mrkdwn"
},
{
"title": "Status",
"short": true,
"value": "${{ steps.get-workflow-info.outputs.conclusion }}",
"type": "mrkdwn"
}
],
"author_name": "${{ github.event.sender.login }}",
"author_link": "${{ github.event.sender.html_url }}",
"author_icon": "${{ github.event.sender.avatar_url }}"
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
- name: Set Pipeline Exit Status
shell: bash
run: |
if [ "${{ steps.get-workflow-info.outputs.workflow_conclusion }}" != "success" ]; then
exit 1
else
exit 0
fi
- name: Done
if: always()
run:
echo "All worflows finished"

View file

@ -443,45 +443,3 @@ jobs:
TWINE_PASSWORD: "${{ steps.get-secrets.outputs.twine-password }}"
run: |
tools pkg pypi-upload artifacts/release/salt-${{ needs.prepare-workflow.outputs.salt-version }}.tar.gz
set-pipeline-exit-status:
# This step is just so we can make github require this step, to pass checks
# on a pull request instead of requiring all
name: Set the ${{ github.workflow }} Pipeline Exit Status
if: always()
runs-on: ubuntu-latest
needs:
- check-requirements
- prepare-workflow
- publish-repositories
- pkg-download-tests
- release
- publish-pypi
- build-ci-deps
steps:
- name: Get workflow information
id: get-workflow-info
uses: im-open/workflow-conclusion@v2
- run: |
# shellcheck disable=SC2129
if [ "${{ steps.get-workflow-info.outputs.conclusion }}" != "success" ]; then
echo 'To restore the release bucket run:' >> "${GITHUB_STEP_SUMMARY}"
echo '```' >> "${GITHUB_STEP_SUMMARY}"
echo 'tools pkg repo restore-previous-releases' >> "${GITHUB_STEP_SUMMARY}"
echo '```' >> "${GITHUB_STEP_SUMMARY}"
fi
- name: Set Pipeline Exit Status
shell: bash
run: |
if [ "${{ steps.get-workflow-info.outputs.workflow_conclusion }}" != "success" ]; then
exit 1
else
exit 0
fi
- name: Done
if: always()
run:
echo "All worflows finished"

View file

@ -1,37 +0,0 @@
name: Retry workflow
run-name: Retry workflows
on:
workflow_dispatch: {}
schedule:
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onschedule
- cron: '*/10 * * * *'
permissions:
contents: read
pull-requests: read
actions: write
jobs:
rerun:
runs-on: ubuntu-latest
environment: ci
steps:
- name: rerun ${{ inputs.run_id }}
env:
GH_REPO: ${{ github.repository }}
GH_TOKEN: ${{ github.token }}
shell: bash
run: |
cat <<"EOF" > jq.cmd
.[]
| select(
( .updatedAt |fromdate ) > ( $d |tonumber ) and .attempt < 5
).databaseId
EOF
gh run ls -w ci.yml --status failure \
--limit=1000 --json databaseId,updatedAt,attempt > json
for i in $(cat json| jq --arg d $((`date -u +%s` - 86400)) "$(cat jq.cmd)"); do
echo Restarting run $i
gh run rerun $i --failed || true
done

View file

@ -2189,110 +2189,3 @@ jobs:
path: artifacts/coverage/html/full
retention-days: 7
if-no-files-found: error
set-pipeline-exit-status:
# This step is just so we can make github require this step, to pass checks
# on a pull request instead of requiring all
name: Set the ${{ github.workflow }} Pipeline Exit Status
if: always()
runs-on: ubuntu-latest
needs:
- workflow-requirements
- trigger-branch-scheduled-builds
- prepare-workflow
- pre-commit
- lint
- build-docs
- build-deps-onedir
- build-salt-onedir
- build-pkgs-src
- combine-all-code-coverage
- build-ci-deps
- windows-2016
- windows-2019
- windows-2022
- macos-12
- macos-13
- macos-13-arm64
- rockylinux-8
- rockylinux-8-arm64
- rockylinux-9
- rockylinux-9-arm64
- amazonlinux-2
- amazonlinux-2-arm64
- amazonlinux-2023
- amazonlinux-2023-arm64
- archlinux-lts
- debian-11
- debian-11-arm64
- debian-12
- debian-12-arm64
- fedora-40
- opensuse-15
- photonos-4
- photonos-4-arm64
- photonos-4-fips
- photonos-4-arm64-fips
- photonos-5
- photonos-5-arm64
- photonos-5-fips
- photonos-5-arm64-fips
- ubuntu-2004
- ubuntu-2004-arm64
- ubuntu-2204
- ubuntu-2204-arm64
- ubuntu-2404
- ubuntu-2404-arm64
- rockylinux-8-pkg-tests
- rockylinux-8-arm64-pkg-tests
- rockylinux-9-pkg-tests
- rockylinux-9-arm64-pkg-tests
- amazonlinux-2-pkg-tests
- amazonlinux-2-arm64-pkg-tests
- amazonlinux-2023-pkg-tests
- amazonlinux-2023-arm64-pkg-tests
- debian-11-pkg-tests
- debian-11-arm64-pkg-tests
- debian-12-pkg-tests
- debian-12-arm64-pkg-tests
- photonos-4-pkg-tests
- photonos-4-arm64-pkg-tests
- photonos-4-pkg-tests-fips
- photonos-4-arm64-pkg-tests-fips
- photonos-5-pkg-tests
- photonos-5-arm64-pkg-tests
- photonos-5-pkg-tests-fips
- photonos-5-arm64-pkg-tests-fips
- ubuntu-2004-pkg-tests
- ubuntu-2004-arm64-pkg-tests
- ubuntu-2204-pkg-tests
- ubuntu-2204-arm64-pkg-tests
- ubuntu-2404-pkg-tests
- ubuntu-2404-arm64-pkg-tests
- macos-12-pkg-tests
- macos-13-pkg-tests
- macos-13-arm64-pkg-tests
- windows-2016-nsis-pkg-tests
- windows-2016-msi-pkg-tests
- windows-2019-nsis-pkg-tests
- windows-2019-msi-pkg-tests
- windows-2022-nsis-pkg-tests
- windows-2022-msi-pkg-tests
steps:
- name: Get workflow information
id: get-workflow-info
uses: im-open/workflow-conclusion@v2
- name: Set Pipeline Exit Status
shell: bash
run: |
if [ "${{ steps.get-workflow-info.outputs.workflow_conclusion }}" != "success" ]; then
exit 1
else
exit 0
fi
- name: Done
if: always()
run:
echo "All worflows finished"

View file

@ -3038,41 +3038,3 @@ jobs:
TWINE_PASSWORD: "${{ steps.get-secrets.outputs.twine-password }}"
run: |
tools pkg pypi-upload --test artifacts/release/salt-${{ needs.prepare-workflow.outputs.salt-version }}.tar.gz
set-pipeline-exit-status:
# This step is just so we can make github require this step, to pass checks
# on a pull request instead of requiring all
name: Set the ${{ github.workflow }} Pipeline Exit Status
if: always()
runs-on: ubuntu-latest
needs:
- check-requirements
- prepare-workflow
- pre-commit
- lint
- build-docs
- build-deps-onedir
- build-salt-onedir
- build-pkgs-src
- publish-repositories
- upload-release-artifacts
- pkg-download-tests
- publish-pypi
steps:
- name: Get workflow information
id: get-workflow-info
uses: im-open/workflow-conclusion@v2
- name: Set Pipeline Exit Status
shell: bash
run: |
if [ "${{ steps.get-workflow-info.outputs.workflow_conclusion }}" != "success" ]; then
exit 1
else
exit 0
fi
- name: Done
if: always()
run:
echo "All worflows finished"

View file

@ -327,50 +327,3 @@ jobs:
<%- endif %>
<%- endblock jobs %>
set-pipeline-exit-status:
# This step is just so we can make github require this step, to pass checks
# on a pull request instead of requiring all
name: Set the ${{ github.workflow }} Pipeline Exit Status
if: always()
runs-on: ubuntu-latest
<%- if workflow_slug == "nightly" %>
environment: <{ workflow_slug }>
<%- endif %>
needs:
<%- for need in prepare_workflow_needs.iter(consume=True) %>
- <{ need }>
<%- endfor %>
<%- for need in conclusion_needs.iter(consume=True) %>
- <{ need }>
<%- endfor %>
<%- for need in test_salt_needs.iter(consume=False) %>
- <{ need }>
<%- endfor %>
<%- for need in test_salt_pkg_needs.iter(consume=False) %>
- <{ need }>
<%- endfor %>
<%- for need in test_repo_needs.iter(consume=True) %>
- <{ need }>
<%- endfor %>
steps:
- name: Get workflow information
id: get-workflow-info
uses: im-open/workflow-conclusion@v2
<%- block set_pipeline_exit_status_extra_steps %>
<%- endblock set_pipeline_exit_status_extra_steps %>
- name: Set Pipeline Exit Status
shell: bash
run: |
if [ "${{ steps.get-workflow-info.outputs.workflow_conclusion }}" != "success" ]; then
exit 1
else
exit 0
fi
- name: Done
if: always()
run:
echo "All worflows finished"

48
.github/workflows/workflow-finished.yml vendored Normal file
View file

@ -0,0 +1,48 @@
name: Workflow Finished
on:
workflow_run:
workflows:
- CI
types:
- completed
permissions:
contents: read
pull-requests: read
actions: write
jobs:
show-context:
name: Show Github Context
runs-on: ubuntu-latest
steps:
- name: event
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
run: |
echo "$GITHUB_CONTEXT"
on-failure:
runs-on: ubuntu-latest
if: ${{ github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.run_attempt < 5 }}
steps:
- run: |
gh run rerun ${{ github.event.workflow_run.id }} --failed
# Branch protection rules require this to run with exit code 0.
set-pipeline-exit-status:
name: Set the ${{ github.event.workflow.name }} Pipeline Exit Status
if: ${{ github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.run_attempt >= 5 }}
runs-on: ubuntu-latest
needs:
- show-context
- on-failure
steps:
- run:
if [ "${{ github.event.workflow_run.conclusion }}" != "success" ]; then
exit 1
else
exit 0
fi