From baebf629412e1f2f4eab45a7102b477d1653dfef Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 29 Mar 2023 14:57:31 -0700 Subject: [PATCH 01/57] Adding support for installing 3006 via bootstrap. Updating stable type to install onedir packages starting in 3006. --- bootstrap-salt.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index df46974..e668c45 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -617,11 +617,20 @@ if [ "$ITYPE" = "git" ]; then elif [ "$ITYPE" = "stable" ]; then if [ "$#" -eq 0 ];then STABLE_REV="latest" + ITYPE="onedir" else - if [ "$(echo "$1" | grep -E '^(latest|1\.6|1\.7|2014\.1|2014\.7|2015\.5|2015\.8|2016\.3|2016\.11|2017\.7|2018\.3|2019\.2|3000|3001|3002|3003|3004|3005)$')" != "" ]; then + if [ "$(echo "$1" | grep -E '^(nightly|latest|3006)$')" != "" ]; then + ONEDIR_REV="$1" + ITYPE="onedir" + shift + elif [ "$(echo "$1" | grep -E '^(3003|3004|3005)$')" != "" ]; then STABLE_REV="$1" shift - elif [ "$(echo "$1" | grep -E '^(2[0-9]*\.[0-9]*\.[0-9]*|[3-9][0-9]{3}(\.[0-9]*)?)$')" != "" ]; then + elif [ "$(echo "$1" | grep -E '^([3-9][6-9]{3}(\.[0-9]*)?)')" != "" ]; then + ONEDIR_REV="minor/$1" + ITYPE="onedir" + shift + elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{3}(\.[0-9]*)?)$')" != "" ]; then # Handle the 3xxx.0 version as 3xxx archive (pin to minor) and strip the fake ".0" suffix STABLE_REV=$(echo "$1" | sed -E 's/^([3-9][0-9]{3})\.0$/\1/') if [ "$(uname)" != "Darwin" ]; then @@ -629,7 +638,7 @@ elif [ "$ITYPE" = "stable" ]; then fi shift else - echo "Unknown stable version: $1 (valid: 1.6, 1.7, 2014.1, 2014.7, 2015.5, 2015.8, 2016.3, 2016.11, 2017.7, 2018.3, 2019.2, 3000, 3001, 3002, 3003, 3004, 3005, latest, \$MAJOR.\$MINOR.\$PATCH until 2019.2, \$MAJOR or \$MAJOR.\$PATCH starting from 3000)" + echo "Unknown stable version: $1 (valid: 3003, 3004, 3005, 3006, latest)" exit 1 fi fi @@ -638,7 +647,7 @@ elif [ "$ITYPE" = "onedir" ]; then if [ "$#" -eq 0 ];then ONEDIR_REV="latest" else - if [ "$(echo "$1" | grep -E '^(nightly|latest|3005)$')" != "" ]; then + if [ "$(echo "$1" | grep -E '^(nightly|latest|3005|3006)$')" != "" ]; then ONEDIR_REV="$1" shift elif [ "$(echo "$1" | grep -E '^(3005(\.[0-9]*)?)')" != "" ]; then @@ -650,7 +659,7 @@ elif [ "$ITYPE" = "onedir" ]; then ONEDIR_REV="minor/$1" shift else - echo "Unknown onedir version: $1 (valid: 3005, latest, nightly.)" + echo "Unknown onedir version: $1 (valid: 3005, 3006, latest, nightly.)" exit 1 fi fi From 1a7a7087f203a5feeae2bd1942810a24fa28ef09 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 29 Mar 2023 15:52:05 -0700 Subject: [PATCH 02/57] Fix REV variable to be ONEDIR_REV to match ITYPE --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index e668c45..8b5bf19 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -616,7 +616,7 @@ if [ "$ITYPE" = "git" ]; then # If doing stable install, check if version specified elif [ "$ITYPE" = "stable" ]; then if [ "$#" -eq 0 ];then - STABLE_REV="latest" + ONEDIR_REV="latest" ITYPE="onedir" else if [ "$(echo "$1" | grep -E '^(nightly|latest|3006)$')" != "" ]; then From 829094e3fcdb4770240a71967d2313a3e2dd416f Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 30 Mar 2023 12:31:47 -0700 Subject: [PATCH 03/57] account for the older key in latest until latest points to 3006 and is using the newer key. --- bootstrap-salt.sh | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 8b5bf19..884a3db 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3087,8 +3087,11 @@ __install_saltstack_ubuntu_onedir_repository() { fi echo "$__REPO_ARCH_DEB $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/salt.list - if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005|nightly)')" != "" ]; then + if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005)')" != "" ]; then __apt_key_fetch "${SALTSTACK_UBUNTU_URL}salt-archive-keyring.gpg" || return 1 + elif [ "$(echo "${ONEDIR_REV}" | grep -E '(latest|nightly)')" != "" ]; then + __apt_key_fetch "${SALTSTACK_UBUNTU_URL}salt-archive-keyring.gpg" || \ + __apt_key_fetch "${SALTSTACK_UBUNTU_URL}SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 else __apt_key_fetch "${SALTSTACK_UBUNTU_URL}SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 fi @@ -3631,8 +3634,11 @@ __install_saltstack_debian_onedir_repository() { fi echo "$__REPO_ARCH_DEB $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/salt.list" - if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005|nightly)')" != "" ]; then + if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005)')" != "" ]; then __apt_key_fetch "${SALTSTACK_DEBIAN_URL}salt-archive-keyring.gpg" || return 1 + elif [ "$(echo "${ONEDIR_REV}" | grep -E '(latest|nightly)')" != "" ]; then + __apt_key_fetch "${SALTSTACK_UBUNTU_URL}salt-archive-keyring.gpg" || \ + __apt_key_fetch "${SALTSTACK_UBUNTU_URL}SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 else __apt_key_fetch "${SALTSTACK_DEBIAN_URL}SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 fi @@ -4519,12 +4525,18 @@ __install_saltstack_rhel_onedir_repository() { if [ "${ONEDIR_REV}" = "nightly" ] ; then base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/\$basearch/" fi - if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005|nightly)')" != "" ]; then + if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005)')" != "" ]; then if [ "${DISTRO_MAJOR_VERSION}" -eq 9 ]; then gpg_key="SALTSTACK-GPG-KEY2.pub" else gpg_key="SALTSTACK-GPG-KEY.pub" fi + elif [ "$(echo "${ONEDIR_REV}" | grep -E '(latest|nightly)')" != "" ]; then + if [ "${DISTRO_MAJOR_VERSION}" -eq 9 ]; then + gpg_key="SALTSTACK-GPG-KEY2.pub SALT-PROJECT-GPG-PUBKEY-2023.pub" + else + gpg_key="SALTSTACK-GPG-KEY.pub SALT-PROJECT-GPG-PUBKEY-2023.pub" + fi else gpg_key="SALT-PROJECT-GPG-PUBKEY-2023.pub" fi @@ -6312,11 +6324,13 @@ install_amazon_linux_ami_2_onedir_deps() { base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/amazon/2/\$basearch/" fi - if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005|nightly)')" != "" ]; then + if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005)')" != "" ]; then gpg_key="${base_url}SALTSTACK-GPG-KEY.pub,${base_url}base/RPM-GPG-KEY-CentOS-7" if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then gpg_key="${base_url}SALTSTACK-GPG-KEY.pub" fi + elif [ "$(echo "${ONEDIR_REV}" | grep -E '(latest|nightly)')" != "" ]; then + gpg_key="SALTSTACK-GPG-KEY.pub SALT-PROJECT-GPG-PUBKEY-2023.pub" else gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub" fi From 88319abee8be09f43e89ccdceb67b36506c61569 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 30 Mar 2023 12:57:20 -0700 Subject: [PATCH 04/57] add some dummy install onedir functions to call the stable version for certain OSes, eg. freebsd, openbsd, smartos. --- bootstrap-salt.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 884a3db..eef7df2 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4442,6 +4442,11 @@ install_fedora_check_services() { return 0 } +install_fedora_onedir() { + install_fedora_stable || return 1 + + return 0 +} # # Ended Fedora Install Functions # @@ -6881,6 +6886,15 @@ install_freebsd_restart_daemons() { service salt_$fname start done } + +install_freebsd_onedir() { +# +# call install_freebsd_stable +# + install_freebsd_stable || return 1 + + return 0 +} # # Ended FreeBSD Install Functions # @@ -6999,6 +7013,14 @@ install_openbsd_restart_daemons() { return 0 } +install_openbsd_onedir() { +# +# Call install_openbsd_stable +# + install_openbsd_stable || return 1 + + return 0 +} # # Ended OpenBSD Install Functions # @@ -7199,6 +7221,14 @@ install_smartos_restart_daemons() { return 0 } +install_smartos_onedir() { +# +# call install_smartos_stable +# + install_smartos_stable || return 1 + + return 0 +} # # Ended SmartOS Install Functions # From 40eb43e2efdfa49b331e9b449e38dd95fe10b32f Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 30 Mar 2023 15:29:56 -0700 Subject: [PATCH 05/57] on RH based systems, check that the updated GPG key exists otherwise fall back to the previous GPG key. --- bootstrap-salt.sh | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index eef7df2..b9c9154 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -897,6 +897,18 @@ __fetch_verify() { return 1 } +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __check_url_exists +# DESCRIPTION: Checks if a URL exists +#---------------------------------------------------------------------------------------------------------------------- +__check_url_exists() { + _URL="$1" + if curl --output /dev/null --silent --fail "${_URL}"; then + return 0 + else + return 1 + fi +} #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __gather_hardware_info # DESCRIPTION: Discover hardware information @@ -3637,8 +3649,8 @@ __install_saltstack_debian_onedir_repository() { if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005)')" != "" ]; then __apt_key_fetch "${SALTSTACK_DEBIAN_URL}salt-archive-keyring.gpg" || return 1 elif [ "$(echo "${ONEDIR_REV}" | grep -E '(latest|nightly)')" != "" ]; then - __apt_key_fetch "${SALTSTACK_UBUNTU_URL}salt-archive-keyring.gpg" || \ - __apt_key_fetch "${SALTSTACK_UBUNTU_URL}SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 + __apt_key_fetch "${SALTSTACK_DEBIAN_URL}salt-archive-keyring.gpg" || \ + __apt_key_fetch "${SALTSTACK_DEBIAN_URL}SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 else __apt_key_fetch "${SALTSTACK_DEBIAN_URL}SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 fi @@ -4537,10 +4549,14 @@ __install_saltstack_rhel_onedir_repository() { gpg_key="SALTSTACK-GPG-KEY.pub" fi elif [ "$(echo "${ONEDIR_REV}" | grep -E '(latest|nightly)')" != "" ]; then - if [ "${DISTRO_MAJOR_VERSION}" -eq 9 ]; then - gpg_key="SALTSTACK-GPG-KEY2.pub SALT-PROJECT-GPG-PUBKEY-2023.pub" + if __check_url_exists "${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub"; then + gpg_key="SALT-PROJECT-GPG-PUBKEY-2023.pub" else - gpg_key="SALTSTACK-GPG-KEY.pub SALT-PROJECT-GPG-PUBKEY-2023.pub" + if [ "${DISTRO_MAJOR_VERSION}" -eq 9 ]; then + gpg_key="SALTSTACK-GPG-KEY2.pub" + else + gpg_key="SALTSTACK-GPG-KEY.pub SALT-PROJECT-GPG-PUBKEY-2023.pub" + fi fi else gpg_key="SALT-PROJECT-GPG-PUBKEY-2023.pub" @@ -6335,7 +6351,11 @@ install_amazon_linux_ami_2_onedir_deps() { gpg_key="${base_url}SALTSTACK-GPG-KEY.pub" fi elif [ "$(echo "${ONEDIR_REV}" | grep -E '(latest|nightly)')" != "" ]; then - gpg_key="SALTSTACK-GPG-KEY.pub SALT-PROJECT-GPG-PUBKEY-2023.pub" + if __check_url_exists "${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub"; then + gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub" + else + gpg_key="${base_url}SALTSTACK-GPG-KEY.pub" + fi else gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub" fi From edd4b6fbe6632164016cf27450ff323f2251f054 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 30 Mar 2023 15:51:38 -0700 Subject: [PATCH 06/57] Ensure STABLE_REV is still set for a couple functions that still use it, eg. Fedora. --- bootstrap-salt.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index b9c9154..2c122b2 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -617,6 +617,7 @@ if [ "$ITYPE" = "git" ]; then elif [ "$ITYPE" = "stable" ]; then if [ "$#" -eq 0 ];then ONEDIR_REV="latest" + STABLE_REV="latest" ITYPE="onedir" else if [ "$(echo "$1" | grep -E '^(nightly|latest|3006)$')" != "" ]; then From 4eaf22fc59754a8e68833f5c1789285f3eba6ea6 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 30 Mar 2023 17:21:59 -0700 Subject: [PATCH 07/57] Don't include SALT-PROJECT-GPG-PUBKEY-2023.pub if it's not found. --- bootstrap-salt.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 2c122b2..1cb87fc 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4556,7 +4556,7 @@ __install_saltstack_rhel_onedir_repository() { if [ "${DISTRO_MAJOR_VERSION}" -eq 9 ]; then gpg_key="SALTSTACK-GPG-KEY2.pub" else - gpg_key="SALTSTACK-GPG-KEY.pub SALT-PROJECT-GPG-PUBKEY-2023.pub" + gpg_key="SALTSTACK-GPG-KEY.pub" fi fi else @@ -6729,6 +6729,12 @@ install_arch_check_services() { return 0 } + +install_arch_linux_onedir() { + install_arch_linux_stable || return 1 + + return 0 +} # # Ended Arch Install Functions # From 499f895baeafd230cb49abf017d08568281e916c Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 31 Mar 2023 12:57:37 -0700 Subject: [PATCH 08/57] Adding +install_macosx_onedir and friends --- bootstrap-salt.sh | 50 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 1cb87fc..261df25 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -617,7 +617,6 @@ if [ "$ITYPE" = "git" ]; then elif [ "$ITYPE" = "stable" ]; then if [ "$#" -eq 0 ];then ONEDIR_REV="latest" - STABLE_REV="latest" ITYPE="onedir" else if [ "$(echo "$1" | grep -E '^(nightly|latest|3006)$')" != "" ]; then @@ -629,6 +628,7 @@ elif [ "$ITYPE" = "stable" ]; then shift elif [ "$(echo "$1" | grep -E '^([3-9][6-9]{3}(\.[0-9]*)?)')" != "" ]; then ONEDIR_REV="minor/$1" + _ONEDIR_REV="$1" ITYPE="onedir" shift elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{3}(\.[0-9]*)?)$')" != "" ]; then @@ -4456,6 +4456,7 @@ install_fedora_check_services() { return 0 } install_fedora_onedir() { + STABLE_REV=$ONEDIR_REV install_fedora_stable || return 1 return 0 @@ -8439,17 +8440,49 @@ __macosx_get_packagesite() { SALTPKGCONFURL="https://${_REPO_URL}/osx/${PKG}" } +__macosx_get_packagesite_onedir() { + DARWIN_ARCH="x86_64" + + __PY_VERSION_REPO="py2" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + __PY_VERSION_REPO="py3" + fi + + if [ "$(echo "$_ONEDIR_REV" | grep -E '^(3005)$')" != "" ]; then + PKG="salt-${_ONEDIR_REV}-macos-${DARWIN_ARCH}.pkg" + SALTPKGCONFURL="https://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/macos/${_ONEDIR_REV}/${PKG}" + else + if [ "$(echo "$_ONEDIR_REV" | grep -E '(salt_rc)$')" != "" ]; then + UNSIGNED="-unsigned" + else + UNSIGNED="" + fi + PKG="salt-${_ONEDIR_REV}-${__PY_VERSION_REPO}-${DARWIN_ARCH}${UNSIGNED}.pkg" + SALTPKGCONFURL="https://${_REPO_URL}/${ONEDIR_REV}/${PKG}" + fi +} + # Using a separate conf step to head for idempotent install... __configure_macosx_pkg_details() { __macosx_get_packagesite || return 1 return 0 } +__configure_macosx_pkg_details_onedir() { + __macosx_get_packagesite_onedir || return 1 + return 0 +} + install_macosx_stable_deps() { __configure_macosx_pkg_details || return 1 return 0 } +install_macosx_onedir_deps() { + __configure_macosx_pkg_details_onedir || return 1 + return 0 +} + install_macosx_git_deps() { install_macosx_stable_deps || return 1 @@ -8496,6 +8529,16 @@ install_macosx_stable() { return 0 } +install_macosx_onedir() { + install_macosx_onedir_deps || return 1 + + __fetch_url "/tmp/${PKG}" "${SALTPKGCONFURL}" || return 1 + + /usr/sbin/installer -pkg "/tmp/${PKG}" -target / || return 1 + + return 0 +} + install_macosx_git() { if [ -n "$_PY_EXE" ]; then @@ -8533,6 +8576,11 @@ install_macosx_stable_post() { return 0 } +install_macosx_onedir_post() { + install_macosx_stable_post || return 1 + return 0 +} + install_macosx_git_post() { install_macosx_stable_post || return 1 return 0 From 07ab70922ca1edc4eb8aba4354b18cb850d71001 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 31 Mar 2023 16:00:31 -0700 Subject: [PATCH 09/57] need a dummy install_fedora_onedir_post that calls install_fedora_stable_post. --- bootstrap-salt.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 261df25..587cec3 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4461,6 +4461,13 @@ install_fedora_onedir() { return 0 } + +install_fedora_onedir_post() { + STABLE_REV=$ONEDIR_REV + install_fedora_stable_post || return 1 + + return 0 +} # # Ended Fedora Install Functions # From 67a0df2dd12578debf9cdfeebd24448884767706 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 31 Mar 2023 16:01:43 -0700 Subject: [PATCH 10/57] need a dummy install_arch_linux_onedir_post that calls install_arch_linux_stable_post. --- bootstrap-salt.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 587cec3..40bae4b 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -6743,6 +6743,12 @@ install_arch_linux_onedir() { return 0 } + +install_arch_linux_onedir_post() { + install_arch_linux_stable_post || return 1 + + return 0 +} # # Ended Arch Install Functions # From 6eacf22f8a9841fb285839575e0a9c5fb79d998c Mon Sep 17 00:00:00 2001 From: "github-actions[bot] on behalf of @garethgreenaway" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 6 Apr 2023 18:02:17 +0000 Subject: [PATCH 11/57] Update README.rst with 2023.04.06 release sha256sum --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 65c5702..ba2e9ee 100644 --- a/README.rst +++ b/README.rst @@ -32,6 +32,7 @@ sum** of the downloaded ``bootstrap-salt.sh`` file. The SHA256 sum of the ``bootstrap-salt.sh`` file, per release, is: +- 2023.04.06: ``994bf7e8bd92fe6d70d291c7562aff299f5651046b4e76dfa506cee0d9bb0843`` - 2022.10.04: ``d0686c2daeed18bb726e58eef75a69afe9ee56a1a23b2c32cd4e87d6005638e2`` - 2022.08.13: ``af922699c1a2bb3b89b6dac04397389999df1b3416b8d0b5c93766412f14c95c`` - 2022.08.12: ``b46f018bbf02f45c6096ab96e9261a9adb3a78ff65092c3976f32ffde909afcb`` From 359a8e80f782f9761826c7cfd624bc05ddb4d7d2 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 11 Apr 2023 14:36:57 +0100 Subject: [PATCH 12/57] Add actionlint to pre-commit Signed-off-by: Pedro Algarvio --- .github/actionlint.yaml | 4 ++++ .pre-commit-config.yaml | 13 +++++++++++ tools/__init__.py | 8 +++++++ tools/pre_commit.py | 51 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+) create mode 100644 .github/actionlint.yaml create mode 100644 tools/__init__.py create mode 100644 tools/pre_commit.py diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..093a4fd --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,4 @@ +self-hosted-runner: + # Labels of self-hosted runner in array of string + labels: + - repo-release diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b4d63be..e382cca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,6 +21,19 @@ repos: hooks: - id: black + - repo: https://github.com/s0undt3ch/python-tools-scripts + rev: "0.12.0" + hooks: + - id: tools + alias: actionlint + name: Lint GitHub Actions Workflows + files: "^.github/workflows/" + types: + - yaml + args: + - pre-commit + - actionlint + - repo: local hooks: - id: generate-actions-workflow diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..f1a2418 --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1,8 @@ +import logging + +import ptscripts + +ptscripts.register_tools_module("tools.pre_commit") + +for name in ("boto3", "botocore", "urllib3"): + logging.getLogger(name).setLevel(logging.INFO) diff --git a/tools/pre_commit.py b/tools/pre_commit.py new file mode 100644 index 0000000..da24bf8 --- /dev/null +++ b/tools/pre_commit.py @@ -0,0 +1,51 @@ +""" +These commands are used by pre-commit. +""" +# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated +from __future__ import annotations + +import logging +import shutil + +from ptscripts import Context, command_group + + +log = logging.getLogger(__name__) + +# Define the command group +cgroup = command_group( + name="pre-commit", help="Pre-Commit Related Commands", description=__doc__ +) + + +@cgroup.command( + name="actionlint", + arguments={ + "files": { + "help": "Files to run actionlint against", + "nargs": "*", + }, + "no_color": { + "help": "Disable colors in output", + }, + }, +) +def actionlint(ctx: Context, files: list[str], no_color: bool = False): + """ + Run `actionlint` + """ + actionlint = shutil.which("actionlint") + if not actionlint: + ctx.warn("Could not find the 'actionlint' binary") + ctx.exit(0) + cmdline = [actionlint] + if no_color is False: + cmdline.append("-color") + shellcheck = shutil.which("shellcheck") + if shellcheck: + cmdline.append(f"-shellcheck={shellcheck}") + pyflakes = shutil.which("pyflakes") + if pyflakes: + cmdline.append(f"-pyflakes={pyflakes}") + ret = ctx.run(*cmdline, *files, check=False) + ctx.exit(ret.returncode) From b51840eda41ff9aabe341209c0175be337dbefc8 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 11 Apr 2023 15:00:55 +0100 Subject: [PATCH 13/57] Use our runners to publish a release. Update S3 bucket in the process. Signed-off-by: Pedro Algarvio --- .github/workflows/release.yml | 356 +++++++++++++++++------ .github/workflows/scripts/cut-release.py | 8 + 2 files changed, 281 insertions(+), 83 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2f6550c..ea06242 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,13 +3,11 @@ name: Cut Release on: workflow_dispatch jobs: - update-develop: - name: Update CHANGELOG.md and bootstrap-salt.sh + + check-requirements: + name: Check Requirements runs-on: ubuntu-latest - if: github.repository == 'saltstack/salt-bootstrap' - permissions: - contents: write # To be able to publish the release - environment: Release + environment: release-check steps: - name: Check For Admin Permission uses: actions-cool/check-user-permission@v2 @@ -17,42 +15,104 @@ jobs: require: admin username: ${{ github.triggering_actor }} - - name: Check Branch Triggering Release + - name: Check Repository run: | - if [ "${{ github.ref_name }}" != "develop" ] - then - echo "This workflow should only be triggered from the develop branch" - exit 1 + echo "Trying to run the release workflow from repository ${{ github.repository }}" + if [ "${{ github.repository }}" != "saltstack/salt-bootstrap" ]; then + echo "Running the release workflow from the ${{ github.repository }} repository is not allowed" + echo "Allowed repository: saltstack/salt-bootstrap" + exit 1 + else + echo "Allowed to release from repository ${{ github.repository }}" fi + - name: Check Branch + run: | + echo "Trying to run the release workflow from branch ${{ github.ref_name }}" + if [ "${{ github.ref_name }}" != "develop" ]; then + echo "Running the release workflow from the ${{ github.ref_name }} branch is not allowed" + echo "Allowed branches: develop" + exit 1 + else + echo "Allowed to release from branch ${{ github.ref_name }}" + fi + + update-develop: + name: Update CHANGELOG.md and bootstrap-salt.sh + runs-on: + - self-hosted + - linux + - repo-release + permissions: + contents: write # To be able to publish the release + environment: release + needs: + - check-requirements + outputs: + release-changes: ${{ steps.update-repo.outputs.release-changes }} + release-version: ${{ steps.update-repo.outputs.release-version }} + steps: - uses: actions/checkout@v3 with: ref: develop repository: ${{ github.repository }} ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} - - name: Update Git Settings - run: | - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot] on behalf of ${{ github.event.sender.login }}" - - - name: Set up Python 3.7 + - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: "3.10" - name: Install Requirements run: | python3 -m pip install requests pre-commit pre-commit install --install-hooks + - name: Setup GnuPG + run: | + sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + GNUPGHOME="$(mktemp -d -p /run/gpg)" + echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + cat < "${GNUPGHOME}/gpg.conf" + batch + no-tty + pinentry-mode loopback + EOF + + - name: Get Secrets + id: get-secrets + env: + SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + run: | + SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text | jq .default_key -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + | gpg --import - + sync + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + sync + rm "$SECRETS_KEY_FILE" + echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + + - name: Configure Git + shell: bash + run: | + git config --global --add safe.directory "$(pwd)" + git config --global user.name "Salt Project Packaging" + git config --global user.email saltproject-packaging@vmware.com + git config --global user.signingkey 64CBBC8173D76B3F + git config --global commit.gpgsign true + - name: Update Repository + id: update-repo env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | python3 .github/workflows/scripts/cut-release.py --repo ${{ github.repository }} - export CUT_RELEASE_VERSION=$(cat .cut_release_version) - echo "CUT_RELEASE_VERSION=${CUT_RELEASE_VERSION}" >> $GITHUB_ENV - name: Show Changes run: | @@ -61,15 +121,16 @@ jobs: - name: Commit Changes run: | - git commit -am "Update develop branch for the ${CUT_RELEASE_VERSION} release" || \ - git commit -am "Update develop branch for the ${CUT_RELEASE_VERSION} release" + git commit -am "Update develop branch for the ${{ steps.update-repo.outputs.release-version }} release" || \ + git commit -am "Update develop branch for the ${{ steps.update-repo.outputs.release-version }} release" - name: Push Changes - uses: ad-m/github-push-action@master + uses: ad-m/github-push-action@b87afee92c6e70ea888be6203a3e9426fda49839 with: - repository: ${{ github.repository }} - branch: develop ssh: true + atomic: true + branch: develop + repository: ${{ github.repository }} - name: Upload Release Details uses: actions/upload-artifact@v3 @@ -81,10 +142,13 @@ jobs: merge-develop-into-stable: name: Merge develop into stable - runs-on: ubuntu-latest - if: github.repository == 'saltstack/salt-bootstrap' - needs: update-develop - environment: Release + runs-on: + - self-hosted + - linux + - repo-release + needs: + - update-develop + environment: release permissions: contents: write # To be able to publish the release steps: @@ -95,64 +159,90 @@ jobs: ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} fetch-depth: 0 - - name: Update Git Settings + - name: Setup GnuPG run: | - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot] on behalf of @${{ github.event.sender.login }}" + sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + GNUPGHOME="$(mktemp -d -p /run/gpg)" + echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + cat < "${GNUPGHOME}/gpg.conf" + batch + no-tty + pinentry-mode loopback + EOF + + - name: Get Secrets + id: get-secrets + env: + SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + run: | + SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text | jq .default_key -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + | gpg --import - + sync + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + sync + rm "$SECRETS_KEY_FILE" + echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + + - name: Configure Git + shell: bash + run: | + git config --global --add safe.directory "$(pwd)" + git config --global user.name "Salt Project Packaging" + git config --global user.email saltproject-packaging@vmware.com + git config --global user.signingkey 64CBBC8173D76B3F + git config --global commit.gpgsign true - name: Download Release Details uses: actions/download-artifact@v3 with: name: release-details - - name: Update Environment - run: | - export CUT_RELEASE_VERSION=$(cat .cut_release_version) - echo "CUT_RELEASE_VERSION=${CUT_RELEASE_VERSION}" >> $GITHUB_ENV - - name: Merge develop into stable run: | - git merge --no-ff -m "Merge develop into stable for ${CUT_RELEASE_VERSION} release" origin/develop || touch .git-conflicts + git merge --no-ff -m "Merge develop into stable for ${{ needs.update-develop.outputs.release-version }} release" origin/develop || touch .git-conflicts if [ -f .git-conflicts ] then git diff for f in $(git status | grep 'both modified' | awk '{ print $3 }') do - git checkout --theirs $f - pre-commit run -av --files $f - git add $f + git checkout --theirs "$f" + pre-commit run -av --files "$f" + git add "$f" done - git commit -a -m "Merge develop into stable for ${CUT_RELEASE_VERSION} release(auto resolving conflicts to the develop version)" + git commit -a -m "Merge develop into stable for ${{ needs.update-develop.outputs.release-version }} release(auto resolving conflicts to the develop version)" fi - - name: Tag Release - uses: mathieudutour/github-tag-action@v6.0 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - custom_tag: ${{ env.CUT_RELEASE_VERSION }} - tag_prefix: "" - create_annotated_tag: true + - name: Tag The ${{ needs.update-develop.outputs.release-version }} Release + run: | + git tag -m "Release ${{ needs.update-develop.outputs.release-version }}" -as ${{ needs.update-develop.outputs.release-version }} - name: Update bootstrap-salt.sh sha256sum's run: | - echo "$(sha256sum bootstrap-salt.sh | awk '{ print $1 }')" > bootstrap-salt.sh.sha256 - echo "$(sha256sum bootstrap-salt.ps1 | awk '{ print $1 }')" > bootstrap-salt.ps1.sha256 + sha256sum bootstrap-salt.sh | awk '{ print $1 }' > bootstrap-salt.sh.sha256 + sha256sum bootstrap-salt.ps1 | awk '{ print $1 }' > bootstrap-salt.ps1.sha256 git commit -a -m "Update sha256 checksums" || git commit -a -m "Update sha256 checksums" - name: Push Changes - uses: ad-m/github-push-action@master + uses: ad-m/github-push-action@b87afee92c6e70ea888be6203a3e9426fda49839 with: - repository: ${{ github.repository }} - branch: stable - tags: true ssh: true + tags: true + atomic: true + branch: stable + repository: ${{ github.repository }} publish-release: name: Create GitHub Release runs-on: ubuntu-latest - if: github.repository == 'saltstack/salt-bootstrap' - needs: merge-develop-into-stable - environment: Release + needs: + - merge-develop-into-stable + environment: release permissions: contents: write # To be able to publish the release steps: @@ -168,8 +258,8 @@ jobs: - name: Update Environment run: | - export CUT_RELEASE_VERSION=$(cat .cut_release_version) - echo "CUT_RELEASE_VERSION=${CUT_RELEASE_VERSION}" >> $GITHUB_ENV + CUT_RELEASE_VERSION=$(cat .cut_release_version) + echo "CUT_RELEASE_VERSION=${CUT_RELEASE_VERSION}" >> "$GITHUB_ENV" - name: Create Github Release uses: softprops/action-gh-release@v1 @@ -194,12 +284,38 @@ jobs: name: release-details failOnError: false + update-s3-bucket: + name: Update S3 Bucket + runs-on: + - self-hosted + - linux + - repo-release + needs: + - publish-release + environment: release + + steps: + - uses: actions/checkout@v3 + with: + ref: stable + repository: ${{ github.repository }} + + - name: Upload Stable Release to S3 + run: | + files="bootstrap-salt.sh bootstrap-salt.sh.sha256 bootstrap-salt.ps1 bootstrap-salt.ps1.sha256" + for fname in $files; do + aws s3 cp "${fname}" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/stable/${fname}" + done + update-develop-checksums: name: Update Release Checksums on Develop - runs-on: ubuntu-latest - if: github.repository == 'saltstack/salt-bootstrap' - needs: publish-release - environment: Release + runs-on: + - self-hosted + - linux + - repo-release + needs: + - publish-release + environment: release permissions: contents: write # For action peter-evans/create-pull-request pull-requests: write # For action peter-evans/create-pull-request @@ -213,8 +329,8 @@ jobs: - name: Get bootstrap-salt.sh on stable branch sha256sum run: | - echo "SH=$(sha256sum bootstrap-salt.sh | awk '{ print $1 }')" >> $GITHUB_ENV - echo "BS_VERSION=$(sh bootstrap-salt.sh -v | awk '{ print $4 }')" >> $GITHUB_ENV + echo "SH=$(sha256sum bootstrap-salt.sh | awk '{ print $1 }')" >> "$GITHUB_ENV" + echo "BS_VERSION=$(sh bootstrap-salt.sh -v | awk '{ print $4 }')" >> "$GITHUB_ENV" - uses: actions/checkout@v3 with: @@ -222,15 +338,49 @@ jobs: repository: ${{ github.repository }} ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} - - name: Update Git Settings + - name: Setup GnuPG run: | - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot] on behalf of @${{ github.event.sender.login }}" + sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + GNUPGHOME="$(mktemp -d -p /run/gpg)" + echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + cat < "${GNUPGHOME}/gpg.conf" + batch + no-tty + pinentry-mode loopback + EOF - - name: Set up Python 3.7 + - name: Get Secrets + id: get-secrets + env: + SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + run: | + SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text | jq .default_key -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + | gpg --import - + sync + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + sync + rm "$SECRETS_KEY_FILE" + echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + + - name: Configure Git + shell: bash + run: | + git config --global --add safe.directory "$(pwd)" + git config --global user.name "Salt Project Packaging" + git config --global user.email saltproject-packaging@vmware.com + git config --global user.signingkey 64CBBC8173D76B3F + git config --global commit.gpgsign true + + - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: "3.10" - name: Update Latest Release on README run: | @@ -247,19 +397,23 @@ jobs: git commit -am "Update README.rst with ${{ env.BS_VERSION }} release sha256sum" - name: Push Changes - uses: ad-m/github-push-action@master + uses: ad-m/github-push-action@b87afee92c6e70ea888be6203a3e9426fda49839 with: - repository: ${{ github.repository }} - branch: develop ssh: true + atomic: true + branch: develop + repository: ${{ github.repository }} salt: name: Update Release on Salt Repo - runs-on: ubuntu-latest - if: github.repository == 'saltstack/salt-bootstrap' - needs: update-develop-checksums - environment: Open PR Against Salt + runs-on: + - self-hosted + - linux + - repo-release + needs: + - update-develop-checksums + environment: release permissions: contents: write # For action peter-evans/create-pull-request pull-requests: write # For action peter-evans/create-pull-request @@ -272,7 +426,7 @@ jobs: - name: Get bootstrap version run: | - echo "BS_VERSION=$(sh bootstrap-salt.sh -v | awk '{ print $4 }')" >> $GITHUB_ENV + echo "BS_VERSION=$(sh bootstrap-salt.sh -v | awk '{ print $4 }')" >> "$GITHUB_ENV" - uses: actions/checkout@v3 with: @@ -281,17 +435,51 @@ jobs: path: salt-checkout token: ${{ secrets.SALT_REPO_WRITE_TOKEN }} - - name: Update Git Settings + - name: Setup GnuPG run: | - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot] on behalf of @${{ github.event.sender.login }}" + sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + GNUPGHOME="$(mktemp -d -p /run/gpg)" + echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + cat < "${GNUPGHOME}/gpg.conf" + batch + no-tty + pinentry-mode loopback + EOF + + - name: Get Secrets + id: get-secrets + env: + SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + run: | + SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text | jq .default_key -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + | gpg --import - + sync + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + sync + rm "$SECRETS_KEY_FILE" + echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + + - name: Configure Git + shell: bash + run: | + git config --global --add safe.directory "$(pwd)" + git config --global user.name "Salt Project Packaging" + git config --global user.email saltproject-packaging@vmware.com + git config --global user.signingkey 64CBBC8173D76B3F + git config --global commit.gpgsign true - name: Update bootstrap script on Salt run: | cp bootstrap-salt.sh salt-checkout/salt/cloud/deploy/bootstrap-salt.sh - name: Create Pull Request Against Develop - uses: peter-evans/create-pull-request@v4 + uses: peter-evans/create-pull-request@v5 with: title: "Update the bootstrap script to v${{ env.BS_VERSION }}" path: salt-checkout @@ -299,3 +487,5 @@ jobs: token: ${{ secrets.SALT_REPO_WRITE_TOKEN }} commit-message: Update the bootstrap script to v${{ env.BS_VERSION }} delete-branch: true + add-paths: | + salt-checkout/salt/cloud/deploy/bootstrap-salt.sh diff --git a/.github/workflows/scripts/cut-release.py b/.github/workflows/scripts/cut-release.py index 4cbda55..569907c 100644 --- a/.github/workflows/scripts/cut-release.py +++ b/.github/workflows/scripts/cut-release.py @@ -155,6 +155,8 @@ def main(): message=f"Unable to generate changelog. HTTP Response:\n{changelog}", ) + github_output = os.environ.get("GITHUB_OUTPUT") + cut_release_version = REPO_ROOT / ".cut_release_version" print( f"* Writing {cut_release_version.relative_to(REPO_ROOT)} ...", @@ -162,6 +164,9 @@ def main(): flush=True, ) cut_release_version.write_text(options.release_tag) + if github_output is not None: + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"release-version={options.release_tag}\n") cut_release_changes = REPO_ROOT / ".cut_release_changes" print( @@ -170,6 +175,9 @@ def main(): flush=True, ) cut_release_changes.write_text(changelog["body"]) + if github_output is not None: + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"release-changes={changelog['body']}\n") print( f"* Updating {changelog_file.relative_to(REPO_ROOT)} ...", From c981eae0cb83f12fa97242c60dd444a02624b9b2 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 11 Apr 2023 15:20:21 +0100 Subject: [PATCH 14/57] Sign release scripts. Nightly develop version of scripts upload. Signed-off-by: Pedro Algarvio --- .github/workflows/nightly.yml | 98 +++++++++++++++++++++++++++++++++++ .github/workflows/release.yml | 36 ++++++++++++- 2 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/nightly.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000..95b5c86 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,98 @@ +name: Nightly S3 Update +run-name: "Nightly S3 (branch: ${{ github.ref_name }})" + +on: + workflow_dispatch: {} + schedule: + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onschedule + - cron: '0 1 * * *' # Every day at 1AM + +jobs: + + workflow-requirements: + name: Check Workflow Requirements + runs-on: ubuntu-latest + outputs: + requirements-met: ${{ steps.check-requirements.outputs.requirements-met }} + steps: + - name: Check For Admin Permission + if: ${{ github.event_name != 'schedule' }} + uses: actions-cool/check-user-permission@v2 + with: + require: admin + username: ${{ github.triggering_actor }} + + - name: Check Requirements + id: check-requirements + run: | + if [ "${{ vars.RUN_SCHEDULED_BUILDS }}" = "1" ]; then + MSG="Running workflow because RUN_SCHEDULED_BUILDS=1" + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" + echo "requirements-met=true" >> "${GITHUB_OUTPUT}" + elif [ "${{ github.event.repository.fork }}" = "true" ]; then + MSG="Not running workflow because ${{ github.repository }} is a fork" + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" + echo "requirements-met=false" >> "${GITHUB_OUTPUT}" + else + MSG="Running workflow because ${{ github.repository }} is not a fork" + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" + echo "requirements-met=true" >> "${GITHUB_OUTPUT}" + fi + + update-s3-bucket: + name: Update S3 Bucket + if: ${{ fromJSON(needs.workflow-requirements.outputs.requirements-met) }} + runs-on: + - self-hosted + - linux + - repo-release + needs: + - workflow-requirements + environment: release + + steps: + - uses: actions/checkout@v3 + + - name: Setup GnuPG + run: | + sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + GNUPGHOME="$(mktemp -d -p /run/gpg)" + echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + cat < "${GNUPGHOME}/gpg.conf" + batch + no-tty + pinentry-mode loopback + EOF + + - name: Get Secrets + id: get-secrets + env: + SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + run: | + SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text | jq .default_key -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + | gpg --import - + sync + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + sync + rm "$SECRETS_KEY_FILE" + echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + + - name: Upload Develop to S3 + run: | + gpg --output SALT-PROJECT-GPG-PUBKEY-2023.gpg --export 64CBBC8173D76B3F + gpg --armor --output SALT-PROJECT-GPG-PUBKEY-2023.pub --export 64CBBC8173D76B3F + files="bootstrap-salt.sh bootstrap-salt.sh.sha256 bootstrap-salt.ps1 bootstrap-salt.ps1.sha256 SALT-PROJECT-GPG-PUBKEY-2023.gpg SALT-PROJECT-GPG-PUBKEY-2023.pub" + for fname in $files; do + gpg --local-user 64CBBC8173D76B3F --output "${fname}.asc" --armor --detach-sign --sign "${fpath}" + aws s3 cp "${fname}" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/${{ github.ref_name }}/${fname}" + aws s3 cp "${fname}.asc" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/${{ github.ref_name }}/${fname}.asc" + done diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ea06242..a9c96e2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -300,11 +300,45 @@ jobs: ref: stable repository: ${{ github.repository }} + - name: Setup GnuPG + run: | + sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + GNUPGHOME="$(mktemp -d -p /run/gpg)" + echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + cat < "${GNUPGHOME}/gpg.conf" + batch + no-tty + pinentry-mode loopback + EOF + + - name: Get Secrets + id: get-secrets + env: + SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + run: | + SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text | jq .default_key -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + | gpg --import - + sync + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + sync + rm "$SECRETS_KEY_FILE" + echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + - name: Upload Stable Release to S3 run: | - files="bootstrap-salt.sh bootstrap-salt.sh.sha256 bootstrap-salt.ps1 bootstrap-salt.ps1.sha256" + gpg --output SALT-PROJECT-GPG-PUBKEY-2023.gpg --export 64CBBC8173D76B3F + gpg --armor --output SALT-PROJECT-GPG-PUBKEY-2023.pub --export 64CBBC8173D76B3F + files="bootstrap-salt.sh bootstrap-salt.sh.sha256 bootstrap-salt.ps1 bootstrap-salt.ps1.sha256 SALT-PROJECT-GPG-PUBKEY-2023.gpg SALT-PROJECT-GPG-PUBKEY-2023.pub" for fname in $files; do + gpg --local-user 64CBBC8173D76B3F --output "${fname}.asc" --armor --detach-sign --sign "${fpath}" aws s3 cp "${fname}" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/stable/${fname}" + aws s3 cp "${fname}.asc" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/stable/${fname}.asc" done update-develop-checksums: From af258021fa9f48e1ade025e2c817078e2e87b30e Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 11 Apr 2023 15:55:22 +0100 Subject: [PATCH 15/57] Some echo's so that we know what we're doing Signed-off-by: Pedro Algarvio --- .github/workflows/nightly.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 95b5c86..72ca6cf 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -88,11 +88,16 @@ jobs: - name: Upload Develop to S3 run: | + echo "Exporting GPG Key" gpg --output SALT-PROJECT-GPG-PUBKEY-2023.gpg --export 64CBBC8173D76B3F + echo "Exporting Armored GPG Key" gpg --armor --output SALT-PROJECT-GPG-PUBKEY-2023.pub --export 64CBBC8173D76B3F - files="bootstrap-salt.sh bootstrap-salt.sh.sha256 bootstrap-salt.ps1 bootstrap-salt.ps1.sha256 SALT-PROJECT-GPG-PUBKEY-2023.gpg SALT-PROJECT-GPG-PUBKEY-2023.pub" - for fname in $files; do + declare -a files=("bootstrap-salt.sh bootstrap-salt.sh.sha256 bootstrap-salt.ps1 bootstrap-salt.ps1.sha256 SALT-PROJECT-GPG-PUBKEY-2023.gpg SALT-PROJECT-GPG-PUBKEY-2023.pub") + for fname in "${files[@]}" + do + echo "GPG Signing ${fpath} ..." gpg --local-user 64CBBC8173D76B3F --output "${fname}.asc" --armor --detach-sign --sign "${fpath}" + echo "Uploading ${fpath} and ${fpath}.asc to S3" aws s3 cp "${fname}" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/${{ github.ref_name }}/${fname}" aws s3 cp "${fname}.asc" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/${{ github.ref_name }}/${fname}.asc" done From 453812a6ea2d6c1dc2f4ca6ca235c1081402a9b0 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 11 Apr 2023 17:16:05 +0100 Subject: [PATCH 16/57] Switch to using `tools` when uploading to S3 Signed-off-by: Pedro Algarvio --- .github/workflows/nightly.yml | 29 +++++---- .github/workflows/release.yml | 29 +++++---- .pre-commit-config.yaml | 57 +++++++++++++++-- requirements/release.in | 3 + requirements/release.txt | 69 +++++++++++++++++++++ tools/__init__.py | 1 + tools/release.py | 113 ++++++++++++++++++++++++++++++++++ tools/utils.py | 80 ++++++++++++++++++++++++ 8 files changed, 351 insertions(+), 30 deletions(-) create mode 100644 requirements/release.in create mode 100644 requirements/release.txt create mode 100644 tools/release.py create mode 100644 tools/utils.py diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 72ca6cf..4bd69c8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -56,6 +56,12 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Get Salt Project GitHub Actions Bot Environment + run: | + TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") + SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) + echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" + - name: Setup GnuPG run: | sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg @@ -86,18 +92,15 @@ jobs: rm "$SECRETS_KEY_FILE" echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install Requirements + run: | + python3 -m pip install -r requirements/release.txt + - name: Upload Develop to S3 run: | - echo "Exporting GPG Key" - gpg --output SALT-PROJECT-GPG-PUBKEY-2023.gpg --export 64CBBC8173D76B3F - echo "Exporting Armored GPG Key" - gpg --armor --output SALT-PROJECT-GPG-PUBKEY-2023.pub --export 64CBBC8173D76B3F - declare -a files=("bootstrap-salt.sh bootstrap-salt.sh.sha256 bootstrap-salt.ps1 bootstrap-salt.ps1.sha256 SALT-PROJECT-GPG-PUBKEY-2023.gpg SALT-PROJECT-GPG-PUBKEY-2023.pub") - for fname in "${files[@]}" - do - echo "GPG Signing ${fpath} ..." - gpg --local-user 64CBBC8173D76B3F --output "${fname}.asc" --armor --detach-sign --sign "${fpath}" - echo "Uploading ${fpath} and ${fpath}.asc to S3" - aws s3 cp "${fname}" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/${{ github.ref_name }}/${fname}" - aws s3 cp "${fname}.asc" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/${{ github.ref_name }}/${fname}.asc" - done + tools release s3-publish --key-id 64CBBC8173D76B3F develop diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a9c96e2..2572614 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,7 @@ jobs: - name: Install Requirements run: | - python3 -m pip install requests pre-commit + python3 -m pip install -r requirements/release.txt pre-commit install --install-hooks - name: Setup GnuPG @@ -296,9 +296,12 @@ jobs: steps: - uses: actions/checkout@v3 - with: - ref: stable - repository: ${{ github.repository }} + + - name: Get Salt Project GitHub Actions Bot Environment + run: | + TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") + SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) + echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" - name: Setup GnuPG run: | @@ -330,16 +333,18 @@ jobs: rm "$SECRETS_KEY_FILE" echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install Requirements + run: | + python3 -m pip install -r requirements/release.txt + - name: Upload Stable Release to S3 run: | - gpg --output SALT-PROJECT-GPG-PUBKEY-2023.gpg --export 64CBBC8173D76B3F - gpg --armor --output SALT-PROJECT-GPG-PUBKEY-2023.pub --export 64CBBC8173D76B3F - files="bootstrap-salt.sh bootstrap-salt.sh.sha256 bootstrap-salt.ps1 bootstrap-salt.ps1.sha256 SALT-PROJECT-GPG-PUBKEY-2023.gpg SALT-PROJECT-GPG-PUBKEY-2023.pub" - for fname in $files; do - gpg --local-user 64CBBC8173D76B3F --output "${fname}.asc" --armor --detach-sign --sign "${fpath}" - aws s3 cp "${fname}" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/stable/${fname}" - aws s3 cp "${fname}.asc" "s3://${{ vars.S3_BUCKET || 'salt-project-prod-salt-artifacts-release' }}/bootstrap/stable/${fname}.asc" - done + tools release s3-publish --key-id 64CBBC8173D76B3F stable update-develop-checksums: name: Update Release Checksums on Develop diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e382cca..655876b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,11 +16,6 @@ repos: hooks: - id: mdformat - - repo: https://github.com/psf/black - rev: 22.6.0 - hooks: - - id: black - - repo: https://github.com/s0undt3ch/python-tools-scripts rev: "0.12.0" hooks: @@ -34,6 +29,58 @@ repos: - pre-commit - actionlint + - repo: https://github.com/jazzband/pip-tools + rev: 6.13.0 + hooks: + - id: pip-compile + files: ^requirements/release\.(in|txt)$ + args: + - requirements/release.in + + - repo: https://github.com/asottile/pyupgrade + rev: v2.37.3 + hooks: + - id: pyupgrade + name: Rewrite Code to be Py3.9+ + args: [--py39-plus] + + - repo: https://github.com/asottile/reorder_python_imports + rev: v3.8.2 + hooks: + - id: reorder-python-imports + args: [--py39-plus] + + - repo: https://github.com/psf/black + rev: 22.6.0 + hooks: + - id: black + args: [] + +# - repo: https://github.com/PyCQA/flake8 +# rev: 3.9.2 +# hooks: +# - id: flake8 +# language_version: python3 +# additional_dependencies: +# - flake8-mypy-fork +# - flake8-docstrings +# - flake8-typing-imports +# +# - repo: https://github.com/pre-commit/mirrors-mypy +# rev: v1.0.0 +# hooks: +# - id: mypy +# alias: mypy-tools +# name: Run mypy against tools +# files: ^tools/.*\.py$ +# #args: [--strict] +# additional_dependencies: +# - attrs +# - rich +# - types-attrs +# - types-pyyaml +# - types-requests + - repo: local hooks: - id: generate-actions-workflow diff --git a/requirements/release.in b/requirements/release.in new file mode 100644 index 0000000..d0d1a73 --- /dev/null +++ b/requirements/release.in @@ -0,0 +1,3 @@ +pre-commit +python-tools-scripts >= 0.12.0 +boto3 diff --git a/requirements/release.txt b/requirements/release.txt new file mode 100644 index 0000000..b4d5a12 --- /dev/null +++ b/requirements/release.txt @@ -0,0 +1,69 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile requirements/release.in +# +attrs==22.2.0 + # via python-tools-scripts +boto3==1.26.110 + # via -r requirements/release.in +botocore==1.29.110 + # via + # boto3 + # s3transfer +certifi==2022.12.7 + # via requests +cfgv==3.3.1 + # via pre-commit +charset-normalizer==3.1.0 + # via requests +distlib==0.3.6 + # via virtualenv +filelock==3.11.0 + # via virtualenv +identify==2.5.22 + # via pre-commit +idna==3.4 + # via requests +jmespath==1.0.1 + # via + # boto3 + # botocore +markdown-it-py==2.2.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +nodeenv==1.7.0 + # via pre-commit +platformdirs==3.2.0 + # via virtualenv +pre-commit==3.2.2 + # via -r requirements/release.in +pygments==2.15.0 + # via rich +python-dateutil==2.8.2 + # via botocore +python-tools-scripts==0.12.0 + # via -r requirements/release.in +pyyaml==6.0 + # via pre-commit +requests==2.28.2 + # via python-tools-scripts +rich==13.3.3 + # via python-tools-scripts +s3transfer==0.6.0 + # via boto3 +six==1.16.0 + # via python-dateutil +typing-extensions==4.5.0 + # via python-tools-scripts +urllib3==1.26.15 + # via + # botocore + # requests +virtualenv==20.21.0 + # via pre-commit + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/tools/__init__.py b/tools/__init__.py index f1a2418..4a820ca 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -3,6 +3,7 @@ import logging import ptscripts ptscripts.register_tools_module("tools.pre_commit") +ptscripts.register_tools_module("tools.release") for name in ("boto3", "botocore", "urllib3"): logging.getLogger(name).setLevel(logging.INFO) diff --git a/tools/release.py b/tools/release.py new file mode 100644 index 0000000..2b2ee7d --- /dev/null +++ b/tools/release.py @@ -0,0 +1,113 @@ +""" +These commands are used to release Salt Bootstrap. +""" +# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated +from __future__ import annotations + +import logging +import sys +from typing import TYPE_CHECKING + +from ptscripts import command_group +from ptscripts import Context + +import tools.utils + +try: + import boto3 +except ImportError: + print( + "\nPlease run 'python -m pip install -r requirements/release.txt'\n", + file=sys.stderr, + flush=True, + ) + raise + +log = logging.getLogger(__name__) + +# Define the command group +release = command_group( + name="release", + help="Release Related Commands", + description=__doc__, +) + + +@release.command( + name="s3-publish", + arguments={ + "branch": { + "help": "The kind of publish to do.", + "choices": ("stable", "develop"), + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + }, +) +def s3_publish(ctx: Context, branch: str, key_id: str = None): + """ + Publish scripts to S3. + """ + if TYPE_CHECKING: + assert key_id + + ctx.info("Preparing upload ...") + s3 = boto3.client("s3") + + ctx.info( + f"Uploading release artifacts to {tools.utils.RELEASE_BUCKET_NAME!r} bucket ..." + ) + paths_to_upload = [ + f"{tools.utils.GPG_KEY_FILENAME}.gpg", + f"{tools.utils.GPG_KEY_FILENAME}.pub", + ] + copy_exclusions = [ + ".asc", + ".gpg", + ".pub", + ".sha256", + ] + + try: + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, tools.utils.REPO_ROOT) + + for fpath in tools.utils.REPO_ROOT.glob("bootstrap-salt.*"): + if fpath.suffix in copy_exclusions: + continue + paths_to_upload.append(fpath.name) + ret = ctx.run( + "sha256sum", + fpath.relative_to(tools.utils.REPO_ROOT), + capture=True, + check=False, + ) + if ret.returncode: + ctx.error( + f"Failed to get the sha256sum of {fpath.relative_to(tools.utils.REPO_ROOT)}" + ) + ctx.exit(1) + shasum_file = fpath.parent / f"{fpath.name}.sha256" + shasum_file.write_bytes(ret.stdout) + paths_to_upload.append(shasum_file.name) + tools.utils.gpg_sign(ctx, key_id, shasum_file) + paths_to_upload.append(f"{shasum_file.name}.asc") + tools.utils.gpg_sign(ctx, key_id, fpath) + paths_to_upload.append(f"{fpath.name}.asc") + + for path in paths_to_upload: + upload_path = f"bootstrap/{branch}/{path}" + size = fpath.stat().st_size + ctx.info(f" {upload_path}") + with tools.utils.create_progress_bar(file_progress=True) as progress: + task = progress.add_task(description="Uploading...", total=size) + s3.upload_file( + fpath, + tools.utils.RELEASE_BUCKET_NAME, + upload_path, + Callback=tools.utils.UpdateProgress(progress, task), + ) + except KeyboardInterrupt: + pass diff --git a/tools/utils.py b/tools/utils.py new file mode 100644 index 0000000..b07beb9 --- /dev/null +++ b/tools/utils.py @@ -0,0 +1,80 @@ +# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated +from __future__ import annotations + +import os +import pathlib + +from ptscripts import Context +from rich.progress import BarColumn +from rich.progress import Column +from rich.progress import DownloadColumn +from rich.progress import Progress +from rich.progress import TextColumn +from rich.progress import TimeRemainingColumn +from rich.progress import TransferSpeedColumn + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +GPG_KEY_FILENAME = "SALT-PROJECT-GPG-PUBKEY-2023" +SPB_ENVIRONMENT = os.environ.get("SPB_ENVIRONMENT") or "prod" +RELEASE_BUCKET_NAME = f"salt-project-{SPB_ENVIRONMENT}-salt-artifacts-release" + + +class UpdateProgress: + def __init__(self, progress, task): + self.progress = progress + self.task = task + + def __call__(self, chunk_size): + self.progress.update(self.task, advance=chunk_size) + + +def create_progress_bar(file_progress: bool = False, **kwargs): + if file_progress: + return Progress( + TextColumn("[progress.description]{task.description}"), + BarColumn(), + DownloadColumn(), + TransferSpeedColumn(), + TextColumn("eta"), + TimeRemainingColumn(), + **kwargs, + ) + return Progress( + TextColumn( + "[progress.description]{task.description}", table_column=Column(ratio=3) + ), + BarColumn(), + expand=True, + **kwargs, + ) + + +def export_gpg_key(ctx: Context, key_id: str, export_path: pathlib.Path): + keyfile_gpg = export_path.joinpath(GPG_KEY_FILENAME).with_suffix(".gpg") + if keyfile_gpg.exists(): + keyfile_gpg.unlink() + ctx.info(f"Exporting GnuPG Key '{key_id}' to {keyfile_gpg} ...") + ctx.run("gpg", "--output", str(keyfile_gpg), "--export", key_id) + keyfile_pub = export_path.joinpath(GPG_KEY_FILENAME).with_suffix(".pub") + if keyfile_pub.exists(): + keyfile_pub.unlink() + ctx.info(f"Exporting GnuPG Key '{key_id}' to {keyfile_pub} ...") + ctx.run("gpg", "--armor", "--output", str(keyfile_pub), "--export", key_id) + + +def gpg_sign(ctx: Context, key_id: str, path: pathlib.Path): + ctx.info(f"GPG Signing '{path}' ...") + signature_fpath = path.parent / f"{path.name}.asc" + if signature_fpath.exists(): + signature_fpath.unlink() + ctx.run( + "gpg", + "--local-user", + key_id, + "--output", + str(signature_fpath), + "--armor", + "--detach-sign", + "--sign", + str(path), + ) From c07acc326e8e87471b7b0a122c84e05746c838af Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 12 Apr 2023 14:19:24 +0100 Subject: [PATCH 17/57] Respect the old directory layout Signed-off-by: Pedro Algarvio --- tools/release.py | 110 +++++++++++++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 38 deletions(-) diff --git a/tools/release.py b/tools/release.py index 2b2ee7d..fa703c4 100644 --- a/tools/release.py +++ b/tools/release.py @@ -5,6 +5,8 @@ These commands are used to release Salt Bootstrap. from __future__ import annotations import logging +import os +import pathlib import sys from typing import TYPE_CHECKING @@ -59,54 +61,86 @@ def s3_publish(ctx: Context, branch: str, key_id: str = None): ctx.info( f"Uploading release artifacts to {tools.utils.RELEASE_BUCKET_NAME!r} bucket ..." ) - paths_to_upload = [ - f"{tools.utils.GPG_KEY_FILENAME}.gpg", - f"{tools.utils.GPG_KEY_FILENAME}.pub", - ] - copy_exclusions = [ - ".asc", - ".gpg", - ".pub", - ".sha256", - ] + upload_files = { + "stable": { + f"{tools.utils.GPG_KEY_FILENAME}.gpg": [ + f"bootstrap/stable/{tools.utils.GPG_KEY_FILENAME}.gpg", + ], + f"{tools.utils.GPG_KEY_FILENAME}.pub": [ + f"bootstrap/stable/{tools.utils.GPG_KEY_FILENAME}.pub", + ], + "bootstrap-salt.sh": [ + "bootstrap/stable/bootstrap-salt.sh", + ], + "bootstrap-salt.sh.sha256": [ + "bootstrap/stable/bootstrap-salt.sh.sha256", + "bootstrap/stable/bootstrap/sha256", + ], + "bootstrap-salt.ps1": [ + "bootstrap/stable/bootstrap-salt.ps1", + ], + "bootstrap-salt.ps1.sha256": [ + "bootstrap/stable/bootstrap-salt.ps1.sha256", + "bootstrap/stable/winbootstrap/sha256", + ], + }, + "develop": { + f"{tools.utils.GPG_KEY_FILENAME}.gpg": [ + f"bootstrap/develop/{tools.utils.GPG_KEY_FILENAME}.gpg", + ], + f"{tools.utils.GPG_KEY_FILENAME}.pub": [ + f"bootstrap/develop/{tools.utils.GPG_KEY_FILENAME}.pub", + ], + "bootstrap-salt.sh": [ + "bootstrap/develop/bootstrap-salt.sh", + "bootstrap/develop/bootstrap/develop", + ], + "bootstrap-salt.sh.sha256": [ + "bootstrap/develop/bootstrap-salt.sh.sha256", + ], + "bootstrap-salt.ps1": [ + "bootstrap/develop/bootstrap-salt.ps1", + "bootstrap/develop/winbootstrap/develop", + ], + "bootstrap-salt.ps1.sha256": [ + "bootstrap/develop/bootstrap-salt.ps1.sha256", + ], + }, + } + + files_to_upload: list[tuple[str, str]] = [] try: # Export the GPG key in use tools.utils.export_gpg_key(ctx, key_id, tools.utils.REPO_ROOT) - - for fpath in tools.utils.REPO_ROOT.glob("bootstrap-salt.*"): - if fpath.suffix in copy_exclusions: - continue - paths_to_upload.append(fpath.name) - ret = ctx.run( - "sha256sum", - fpath.relative_to(tools.utils.REPO_ROOT), - capture=True, - check=False, - ) - if ret.returncode: - ctx.error( - f"Failed to get the sha256sum of {fpath.relative_to(tools.utils.REPO_ROOT)}" + for lpath, rpaths in upload_files[branch].items(): + ctx.info(f"Processing {lpath} ...") + if lpath.endswith(".sha256") and not os.path.exists(lpath): + ret = ctx.run( + "sha256sum", + lpath.replace(".sha256", ""), + capture=True, + check=False, ) - ctx.exit(1) - shasum_file = fpath.parent / f"{fpath.name}.sha256" - shasum_file.write_bytes(ret.stdout) - paths_to_upload.append(shasum_file.name) - tools.utils.gpg_sign(ctx, key_id, shasum_file) - paths_to_upload.append(f"{shasum_file.name}.asc") - tools.utils.gpg_sign(ctx, key_id, fpath) - paths_to_upload.append(f"{fpath.name}.asc") + if ret.returncode: + ctx.error(f"Failed to get the sha256sum of {lpath}") + ctx.exit(1) + pathlib.Path(lpath).write_bytes(ret.stdout) + for rpath in rpaths: + files_to_upload.append((lpath, rpath)) + if not lpath.endswith((".gpg", ".pub")): + tools.utils.gpg_sign(ctx, key_id, pathlib.Path(lpath)) + files_to_upload.append((f"{lpath}.asc", f"{rpaths[0]}.asc")) - for path in paths_to_upload: - upload_path = f"bootstrap/{branch}/{path}" - size = fpath.stat().st_size - ctx.info(f" {upload_path}") + for lpath, rpath in sorted(files_to_upload): + size = pathlib.Path(lpath).stat().st_size + ctx.info(f" Uploading {lpath} -> {rpath}") with tools.utils.create_progress_bar(file_progress=True) as progress: task = progress.add_task(description="Uploading...", total=size) s3.upload_file( - fpath, + lpath, tools.utils.RELEASE_BUCKET_NAME, - upload_path, + rpath, Callback=tools.utils.UpdateProgress(progress, task), ) except KeyboardInterrupt: From 0184635370363927e0de35dc8823116cdbea8d11 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 12 Apr 2023 14:24:59 +0100 Subject: [PATCH 18/57] Use the installed python on the runners Signed-off-by: Pedro Algarvio --- .github/workflows/nightly.yml | 5 ----- .github/workflows/release.yml | 15 --------------- 2 files changed, 20 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 4bd69c8..d043786 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -92,11 +92,6 @@ jobs: rm "$SECRETS_KEY_FILE" echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Install Requirements run: | python3 -m pip install -r requirements/release.txt diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2572614..7734c9d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,11 +58,6 @@ jobs: repository: ${{ github.repository }} ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Install Requirements run: | python3 -m pip install -r requirements/release.txt @@ -333,11 +328,6 @@ jobs: rm "$SECRETS_KEY_FILE" echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Install Requirements run: | python3 -m pip install -r requirements/release.txt @@ -416,11 +406,6 @@ jobs: git config --global user.signingkey 64CBBC8173D76B3F git config --global commit.gpgsign true - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Update Latest Release on README run: | python3 .github/workflows/scripts/update-release-shasum.py ${{ env.BS_VERSION }} ${{ env.SH }} From 2a4f2ad3f66ac169192df94d9a2296fe7e7fee68 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 12 Apr 2023 14:37:32 +0100 Subject: [PATCH 19/57] Allow testing releases from forks Signed-off-by: Pedro Algarvio --- .github/workflows/release.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7734c9d..211a8eb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,13 +17,25 @@ jobs: - name: Check Repository run: | + if [ "${{ vars.RUN_RELEASE_BUILDS }}" = "1" ]; then + MSG="Running workflow because RUN_RELEASE_BUILDS=1" + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi echo "Trying to run the release workflow from repository ${{ github.repository }}" if [ "${{ github.repository }}" != "saltstack/salt-bootstrap" ]; then - echo "Running the release workflow from the ${{ github.repository }} repository is not allowed" - echo "Allowed repository: saltstack/salt-bootstrap" + MSG="Running the release workflow from the ${{ github.repository }} repository is not allowed" + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" + MSG="Allowed repository: saltstack/salt-bootstrap" + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" exit 1 else - echo "Allowed to release from repository ${{ github.repository }}" + MSG="Allowed to release from repository ${{ github.repository }}" + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" fi - name: Check Branch From 3d3c7c7a9fbeb0d823696841fb3c6437e862b0df Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 12 Apr 2023 15:25:31 +0100 Subject: [PATCH 20/57] Fix opening PR against the Salt repository Signed-off-by: Pedro Algarvio --- .github/workflows/release.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 211a8eb..f89df89 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -514,6 +514,11 @@ jobs: run: | cp bootstrap-salt.sh salt-checkout/salt/cloud/deploy/bootstrap-salt.sh + - name: Commit Changes + working-directory: salt-checkout/ + run: | + git commit -am "Update the bootstrap script to v${{ env.BS_VERSION }}" + - name: Create Pull Request Against Develop uses: peter-evans/create-pull-request@v5 with: @@ -521,7 +526,8 @@ jobs: path: salt-checkout base: master token: ${{ secrets.SALT_REPO_WRITE_TOKEN }} + author: "Salt Project Packaging " + committer: "Salt Project Packaging " commit-message: Update the bootstrap script to v${{ env.BS_VERSION }} + signoff: true delete-branch: true - add-paths: | - salt-checkout/salt/cloud/deploy/bootstrap-salt.sh From 6c6291bdd593d5c71ca038a1abaa4ca5df5e086e Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 12 Apr 2023 15:40:46 +0100 Subject: [PATCH 21/57] Don't rely on GH runners during the release process once started. Signed-off-by: Pedro Algarvio --- .github/workflows/release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f89df89..b6d8f21 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -246,7 +246,9 @@ jobs: publish-release: name: Create GitHub Release - runs-on: ubuntu-latest + runs-on: + - self-hosted + - linux needs: - merge-develop-into-stable environment: release From ce4a0719430bff9021dec328f35ee91bf56be4ef Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 12 Apr 2023 15:42:17 +0100 Subject: [PATCH 22/57] Bump to `geekyeggo/delete-artifact@v2` Signed-off-by: Pedro Algarvio --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b6d8f21..05c8c43 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -288,7 +288,7 @@ jobs: LICENSE - name: Delete Release Details Artifact - uses: geekyeggo/delete-artifact@v1 + uses: geekyeggo/delete-artifact@v2 with: name: release-details failOnError: false From 779cfbf55dbf080db3635f9f8f5b0860f19feea5 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 13 Apr 2023 18:58:29 -0700 Subject: [PATCH 23/57] first pass at parsing the repo.json for Mac when using onedir and latest. --- bootstrap-salt.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 40bae4b..a429804 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -8453,6 +8453,22 @@ __macosx_get_packagesite() { SALTPKGCONFURL="https://${_REPO_URL}/osx/${PKG}" } +__parse_repo_json_python() { + + # Using latest, grab the right + # version from the repo.json + _ONEDIR_REV=$(python - <<-EOF +import json, urllib.request +url = "https://repo.saltproject.io/salt/py3/macos/repo.json" +response = urllib.request.urlopen(url) +data = json.loads(response.read()) +version = data['latest'][list(data['latest'].keys())[0]]['version'] +print(version) +EOF +) + +} + __macosx_get_packagesite_onedir() { DARWIN_ARCH="x86_64" @@ -8470,6 +8486,9 @@ __macosx_get_packagesite_onedir() { else UNSIGNED="" fi + if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then + __parse_repo_json_python || return 1 + fi PKG="salt-${_ONEDIR_REV}-${__PY_VERSION_REPO}-${DARWIN_ARCH}${UNSIGNED}.pkg" SALTPKGCONFURL="https://${_REPO_URL}/${ONEDIR_REV}/${PKG}" fi From f976efc5702cf1238b682a1924b46ac39558d04c Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 13 Apr 2023 21:55:25 -0700 Subject: [PATCH 24/57] different approach to setting the _ONEDIR_REV from the parse repo.json function --- bootstrap-salt.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index a429804..9cb94ea 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -8457,7 +8457,7 @@ __parse_repo_json_python() { # Using latest, grab the right # version from the repo.json - _ONEDIR_REV=$(python - <<-EOF + _JSON_LATEST_VERSION=$(python - <<-EOF import json, urllib.request url = "https://repo.saltproject.io/salt/py3/macos/repo.json" response = urllib.request.urlopen(url) @@ -8466,7 +8466,7 @@ version = data['latest'][list(data['latest'].keys())[0]]['version'] print(version) EOF ) - +echo "${_JSON_LATEST_VERSION}" } __macosx_get_packagesite_onedir() { @@ -8487,7 +8487,7 @@ __macosx_get_packagesite_onedir() { UNSIGNED="" fi if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then - __parse_repo_json_python || return 1 + _ONEDIR_REV=$(__parse_repo_json_python) fi PKG="salt-${_ONEDIR_REV}-${__PY_VERSION_REPO}-${DARWIN_ARCH}${UNSIGNED}.pkg" SALTPKGCONFURL="https://${_REPO_URL}/${ONEDIR_REV}/${PKG}" From b03a99ee53bf2b26d3cdfadc2f0f4498394a0e88 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 14 Apr 2023 05:18:17 -0700 Subject: [PATCH 25/57] Need _ONEDIR_REV defined in a couple more places. --- bootstrap-salt.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 9cb94ea..db58cdd 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -617,10 +617,12 @@ if [ "$ITYPE" = "git" ]; then elif [ "$ITYPE" = "stable" ]; then if [ "$#" -eq 0 ];then ONEDIR_REV="latest" + _ONEDIR_REV="$1" ITYPE="onedir" else if [ "$(echo "$1" | grep -E '^(nightly|latest|3006)$')" != "" ]; then ONEDIR_REV="$1" + _ONEDIR_REV="$1" ITYPE="onedir" shift elif [ "$(echo "$1" | grep -E '^(3003|3004|3005)$')" != "" ]; then From d0032c74951b8c63936ac692559af9f8705c27ba Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 14 Apr 2023 05:33:12 -0700 Subject: [PATCH 26/57] fixing SALTPKGCONFURL, same one for onedir locations just populated with different values for different variables. --- bootstrap-salt.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index db58cdd..60778cd 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -8481,7 +8481,6 @@ __macosx_get_packagesite_onedir() { if [ "$(echo "$_ONEDIR_REV" | grep -E '^(3005)$')" != "" ]; then PKG="salt-${_ONEDIR_REV}-macos-${DARWIN_ARCH}.pkg" - SALTPKGCONFURL="https://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/macos/${_ONEDIR_REV}/${PKG}" else if [ "$(echo "$_ONEDIR_REV" | grep -E '(salt_rc)$')" != "" ]; then UNSIGNED="-unsigned" @@ -8492,8 +8491,8 @@ __macosx_get_packagesite_onedir() { _ONEDIR_REV=$(__parse_repo_json_python) fi PKG="salt-${_ONEDIR_REV}-${__PY_VERSION_REPO}-${DARWIN_ARCH}${UNSIGNED}.pkg" - SALTPKGCONFURL="https://${_REPO_URL}/${ONEDIR_REV}/${PKG}" fi + SALTPKGCONFURL="https://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/macos/${_ONEDIR_REV}/${PKG}" } # Using a separate conf step to head for idempotent install... From efbd416607d610bfab1688ee95176e0d3cc9cda0 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 14 Apr 2023 05:46:31 -0700 Subject: [PATCH 27/57] parse json first, since different versions have different naming schemes and we need to parse that part afterwards --- bootstrap-salt.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 60778cd..7a749dc 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -8479,6 +8479,9 @@ __macosx_get_packagesite_onedir() { __PY_VERSION_REPO="py3" fi + if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then + _ONEDIR_REV=$(__parse_repo_json_python) + fi if [ "$(echo "$_ONEDIR_REV" | grep -E '^(3005)$')" != "" ]; then PKG="salt-${_ONEDIR_REV}-macos-${DARWIN_ARCH}.pkg" else @@ -8487,9 +8490,6 @@ __macosx_get_packagesite_onedir() { else UNSIGNED="" fi - if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then - _ONEDIR_REV=$(__parse_repo_json_python) - fi PKG="salt-${_ONEDIR_REV}-${__PY_VERSION_REPO}-${DARWIN_ARCH}${UNSIGNED}.pkg" fi SALTPKGCONFURL="https://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/macos/${_ONEDIR_REV}/${PKG}" From 0f39aca8c103d48d68c09f145800ce345b650e88 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 14 Apr 2023 05:56:44 -0700 Subject: [PATCH 28/57] Fixing Arch by adding additional onedir functions to call stable functions. --- bootstrap-salt.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 7a749dc..bd4a01e 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -6587,6 +6587,10 @@ install_arch_linux_git_deps() { return 0 } +install_arch_linux_onedir_deps() { + install_arch_linux_stable_deps || return 1 +} + install_arch_linux_stable() { # Pacman does not resolve dependencies on outdated versions # They always need to be updated @@ -6747,7 +6751,7 @@ install_arch_linux_onedir() { } install_arch_linux_onedir_post() { - install_arch_linux_stable_post || return 1 + install_arch_linux_post || return 1 return 0 } From 4a74744f069468ff497aa418b6de19978e0e84fa Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 14 Apr 2023 06:21:35 -0700 Subject: [PATCH 29/57] Remove end of line from grep for 3005. --- bootstrap-salt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index bd4a01e..359ba50 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -8486,10 +8486,10 @@ __macosx_get_packagesite_onedir() { if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then _ONEDIR_REV=$(__parse_repo_json_python) fi - if [ "$(echo "$_ONEDIR_REV" | grep -E '^(3005)$')" != "" ]; then + if [ "$(echo "$_ONEDIR_REV" | grep -E '^(3005)')" != "" ]; then PKG="salt-${_ONEDIR_REV}-macos-${DARWIN_ARCH}.pkg" else - if [ "$(echo "$_ONEDIR_REV" | grep -E '(salt_rc)$')" != "" ]; then + if [ "$(echo "$_ONEDIR_REV" | grep -E '(salt_rc)')" != "" ]; then UNSIGNED="-unsigned" else UNSIGNED="" From b57d37475e99fee4dbed45dd0e87804817b77c01 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 14 Apr 2023 07:08:28 -0700 Subject: [PATCH 30/57] removing python 3.6 related code which is no longer needed. --- bootstrap-salt.sh | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 359ba50..157f465 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -7460,6 +7460,10 @@ install_opensuse_git_deps() { return 0 } +install_opensuse_onedir_deps() { + install_opensuse_stable_deps || return 1 +} + install_opensuse_stable() { __PACKAGES="" @@ -7492,6 +7496,10 @@ install_opensuse_git() { return 0 } +install_opensuse_onedir() { + install_opensuse_stable || return 1 +} + install_opensuse_stable_post() { for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -7561,6 +7569,10 @@ install_opensuse_git_post() { return 0 } +install_opensuse_onedir_post() { + install_opensuse_stable_post || return 1 +} + install_opensuse_restart_daemons() { [ $_START_DAEMONS -eq $BS_FALSE ] && return @@ -8066,11 +8078,6 @@ __gentoo_pre_dep() { mkdir /etc/portage fi - # Enable Python 3.6 target for pre Neon Salt release - if echo "${STABLE_REV}" | grep -q "2019" || [ "${ITYPE}" = "git" ] && [ "${_POST_NEON_INSTALL}" -eq $BS_FALSE ]; then - EXTRA_PYTHON_TARGET=python3_6 - fi - # Enable Python 3.7 target for Salt Neon using GIT if [ "${ITYPE}" = "git" ] && [ "${GIT_REV}" = "v3000" ]; then EXTRA_PYTHON_TARGET=python3_7 @@ -8216,6 +8223,12 @@ install_gentoo_git() { return 0 } +install_gentoo_onedir() { + + install_gentoo_stable || return 1 + +} + install_gentoo_post() { for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -8305,6 +8318,12 @@ _eof return 0 } +install_gentoo_onedir_post() { + + install_gentoo_post || return 1 + +} + install_gentoo_restart_daemons() { [ $_START_DAEMONS -eq $BS_FALSE ] && return From 1aa72d9c2d3738ff39e6d0a58ddbd546bdf5097d Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 14 Apr 2023 08:14:00 -0700 Subject: [PATCH 31/57] Need to set STABLE_REV to ONEDIR_REV when install_gentoo_onedir is called. --- bootstrap-salt.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 157f465..9036288 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -8224,9 +8224,8 @@ install_gentoo_git() { } install_gentoo_onedir() { - + STABLE_REV=${ONEDIR_REV} install_gentoo_stable || return 1 - } install_gentoo_post() { @@ -8319,9 +8318,7 @@ _eof } install_gentoo_onedir_post() { - install_gentoo_post || return 1 - } install_gentoo_restart_daemons() { From be030c1958c9e2d06c88dc4b40072eb06c3f2413 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 14 Apr 2023 09:25:04 -0700 Subject: [PATCH 32/57] suggested fixes --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 9036288..a867fc7 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -8484,7 +8484,7 @@ import json, urllib.request url = "https://repo.saltproject.io/salt/py3/macos/repo.json" response = urllib.request.urlopen(url) data = json.loads(response.read()) -version = data['latest'][list(data['latest'].keys())[0]]['version'] +version = data['latest'][list(data['latest'])[0]]['version'] print(version) EOF ) From 2af0e6c0b6a017b5719b2b2ce4b0d45791b7e19d Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 14 Apr 2023 09:27:39 -0700 Subject: [PATCH 33/57] All Mac packages are now signed. --- bootstrap-salt.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index a867fc7..2a7d2ff 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -8505,12 +8505,7 @@ __macosx_get_packagesite_onedir() { if [ "$(echo "$_ONEDIR_REV" | grep -E '^(3005)')" != "" ]; then PKG="salt-${_ONEDIR_REV}-macos-${DARWIN_ARCH}.pkg" else - if [ "$(echo "$_ONEDIR_REV" | grep -E '(salt_rc)')" != "" ]; then - UNSIGNED="-unsigned" - else - UNSIGNED="" - fi - PKG="salt-${_ONEDIR_REV}-${__PY_VERSION_REPO}-${DARWIN_ARCH}${UNSIGNED}.pkg" + PKG="salt-${_ONEDIR_REV}-${__PY_VERSION_REPO}-${DARWIN_ARCH}.pkg" fi SALTPKGCONFURL="https://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/macos/${_ONEDIR_REV}/${PKG}" } From 90cc157b9d07a4d203a8296cec7eb1c4b9589491 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 18 Apr 2023 16:53:29 -0700 Subject: [PATCH 34/57] Adding 3006 to tests. --- .github/workflows/ci.yml | 52 ++++++++++++------------- .github/workflows/templates/generate.py | 21 ++++------ 2 files changed, 33 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e01c95..2a649d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,7 +173,7 @@ jobs: display-name: macOS 10.15 timeout: 20 runs-on: macos-10.15 - instances: '["stable-3003", "stable-3004", "stable-3005", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "stable-3006", "latest"]' macos-11: @@ -188,7 +188,7 @@ jobs: display-name: macOS 11 timeout: 20 runs-on: macos-11 - instances: '["stable-3003", "stable-3004", "stable-3005", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "stable-3006", "latest"]' macos-12: @@ -203,7 +203,7 @@ jobs: display-name: macOS 12 timeout: 20 runs-on: macos-12 - instances: '["stable-3003", "stable-3004", "stable-3005", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "stable-3006", "latest"]' @@ -219,7 +219,7 @@ jobs: display-name: Windows 2019 timeout: 20 runs-on: windows-2019 - instances: '["stable-3003", "stable-3004", "stable-3005", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "stable-3006", "latest"]' windows-2022: @@ -234,7 +234,7 @@ jobs: display-name: Windows 2022 timeout: 20 runs-on: windows-2022 - instances: '["stable-3003", "stable-3004", "stable-3005", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "stable-3006", "latest"]' @@ -249,7 +249,7 @@ jobs: distro-slug: almalinux-8 display-name: AlmaLinux 8 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest", "onedir-nightly"]' almalinux-9: @@ -263,7 +263,7 @@ jobs: distro-slug: almalinux-9 display-name: AlmaLinux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-master", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "onedir-nightly"]' amazon-2: @@ -277,7 +277,7 @@ jobs: distro-slug: amazon-2 display-name: Amazon 2 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' arch: @@ -291,7 +291,7 @@ jobs: distro-slug: arch display-name: Arch timeout: 20 - instances: '["git-master", "latest"]' + instances: '["stable-3006", "git-3006", "git-master", "latest"]' centos-7: @@ -305,7 +305,7 @@ jobs: distro-slug: centos-7 display-name: CentOS 7 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest", "onedir-nightly"]' centos-stream8: @@ -319,7 +319,7 @@ jobs: distro-slug: centos-stream8 display-name: CentOS Stream 8 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest", "onedir-nightly"]' centos-stream9: @@ -333,7 +333,7 @@ jobs: distro-slug: centos-stream9 display-name: CentOS Stream 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-master", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "onedir-nightly"]' debian-10: @@ -347,7 +347,7 @@ jobs: distro-slug: debian-10 display-name: Debian 10 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' debian-11: @@ -361,7 +361,7 @@ jobs: distro-slug: debian-11 display-name: Debian 11 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' fedora-35: @@ -375,7 +375,7 @@ jobs: distro-slug: fedora-35 display-name: Fedora 35 timeout: 20 - instances: '["git-master", "latest"]' + instances: '["git-3006", "git-master", "latest"]' fedora-36: @@ -389,7 +389,7 @@ jobs: distro-slug: fedora-36 display-name: Fedora 36 timeout: 20 - instances: '["git-master", "latest"]' + instances: '["git-3006", "git-master", "latest"]' fedora-37: @@ -431,7 +431,7 @@ jobs: distro-slug: gentoo display-name: Gentoo timeout: 90 - instances: '["git-master", "latest"]' + instances: '["stable-3006", "git-3006", "git-master", "latest"]' gentoo-systemd: @@ -445,7 +445,7 @@ jobs: distro-slug: gentoo-systemd display-name: Gentoo (systemd) timeout: 90 - instances: '["git-master", "latest"]' + instances: '["stable-3006", "git-3006", "git-master", "latest"]' opensuse-15: @@ -459,7 +459,7 @@ jobs: distro-slug: opensuse-15 display-name: Opensuse 15 timeout: 20 - instances: '["latest"]' + instances: '["stable-3006", "latest"]' opensuse-tumbleweed: @@ -473,7 +473,7 @@ jobs: distro-slug: opensuse-tumbleweed display-name: Opensuse Tumbleweed timeout: 20 - instances: '["git-master", "latest"]' + instances: '["stable-3006", "git-3006", "git-master", "latest"]' oraclelinux-7: @@ -487,7 +487,7 @@ jobs: distro-slug: oraclelinux-7 display-name: Oracle Linux 7 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest", "onedir-nightly"]' oraclelinux-8: @@ -501,7 +501,7 @@ jobs: distro-slug: oraclelinux-8 display-name: Oracle Linux 8 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest", "onedir-nightly"]' rockylinux-8: @@ -515,7 +515,7 @@ jobs: distro-slug: rockylinux-8 display-name: Rocky Linux 8 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest", "onedir-nightly"]' rockylinux-9: @@ -529,7 +529,7 @@ jobs: distro-slug: rockylinux-9 display-name: Rocky Linux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-master", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "onedir-nightly"]' ubuntu-2004: @@ -543,7 +543,7 @@ jobs: distro-slug: ubuntu-2004 display-name: Ubuntu 20.04 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' ubuntu-2204: @@ -557,7 +557,7 @@ jobs: distro-slug: ubuntu-2204 display-name: Ubuntu 22.04 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' set-pipeline-exit-status: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index a966c17..525eb7b 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -1,9 +1,8 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- -import os -import json -import pathlib import datetime +import json +import os +import pathlib os.chdir(os.path.abspath(os.path.dirname(__file__))) @@ -215,28 +214,26 @@ SALT_VERSIONS = [ "3003", "3004", "3005", + "3006", "master", "latest", "nightly", - "3006-0rc1", - "3006-0rc2", ] ONEDIR_SALT_VERSIONS = [ "3005", + "3006", "latest", "nightly", ] -ONEDIR_RC_SALT_VERSIONS = [ - "3006-0rc1", - "3006-0rc2", -] +ONEDIR_RC_SALT_VERSIONS = [] VERSION_DISPLAY_NAMES = { "3003": "v3003", "3004": "v3004", "3005": "v3005", + "3006": "v3006", "master": "Master", "latest": "Latest", "nightly": "Nightly", @@ -245,14 +242,10 @@ VERSION_DISPLAY_NAMES = { STABLE_VERSION_BLACKLIST = [ "master", "nightly", - "3006-0rc1", - "3006-0rc2", ] GIT_VERSION_BLACKLIST = [ "nightly", - "3006-0rc1", - "3006-0rc2", ] # TODO: Revert the commit relating to this section, once the Git-based builds From 27eef7d396616fad400166a0d580cae10847044d Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 18 Apr 2023 17:19:34 -0700 Subject: [PATCH 35/57] Update to MacOS latest install. Need to install procps on onedir centos installs. --- bootstrap-salt.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index f31ad3d..fdff6c6 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4963,6 +4963,8 @@ install_centos_onedir_deps() { __PACKAGES="yum-utils chkconfig" fi + __PACKAGES="${__PACKAGES} procps" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -8507,7 +8509,7 @@ __macosx_get_packagesite_onedir() { else PKG="salt-${_ONEDIR_REV}-${__PY_VERSION_REPO}-${DARWIN_ARCH}.pkg" fi - SALTPKGCONFURL="https://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/macos/${_ONEDIR_REV}/${PKG}" + SALTPKGCONFURL="https://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/macos/minor/${_ONEDIR_REV}/${PKG}" } # Using a separate conf step to head for idempotent install... From ef70886fd06c6234d37ae0031805c7a8e835d7ea Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 18 Apr 2023 17:32:39 -0700 Subject: [PATCH 36/57] removing check for different GPG keys for latest & nightly now that 3006 is available with the new GPG format. --- bootstrap-salt.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index fdff6c6..87a83ee 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4559,16 +4559,6 @@ __install_saltstack_rhel_onedir_repository() { else gpg_key="SALTSTACK-GPG-KEY.pub" fi - elif [ "$(echo "${ONEDIR_REV}" | grep -E '(latest|nightly)')" != "" ]; then - if __check_url_exists "${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub"; then - gpg_key="SALT-PROJECT-GPG-PUBKEY-2023.pub" - else - if [ "${DISTRO_MAJOR_VERSION}" -eq 9 ]; then - gpg_key="SALTSTACK-GPG-KEY2.pub" - else - gpg_key="SALTSTACK-GPG-KEY.pub" - fi - fi else gpg_key="SALT-PROJECT-GPG-PUBKEY-2023.pub" fi From 717db99ef4865e0973e461ae47c13c5eca6fd796 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 18 Apr 2023 18:00:00 -0700 Subject: [PATCH 37/57] Adding 3006.0 to kitchen.windows.yml and kitchen.macos.yml --- kitchen.macos.yml | 3 +++ kitchen.windows.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/kitchen.macos.yml b/kitchen.macos.yml index 3504f2e..d74b5fd 100644 --- a/kitchen.macos.yml +++ b/kitchen.macos.yml @@ -28,6 +28,9 @@ suites: - name: stable-3005 provisioner: salt_version: 3005.1 + - name: stable-3006 + provisioner: + salt_version: 3006.0 - name: latest provisioner: salt_version: latest diff --git a/kitchen.windows.yml b/kitchen.windows.yml index f68f4b1..fe41f5b 100644 --- a/kitchen.windows.yml +++ b/kitchen.windows.yml @@ -26,6 +26,9 @@ suites: - name: stable-3005 provisioner: salt_version: 3005.1-1 + - name: stable-3006 + provisioner: + salt_version: 3006.0 - name: latest provisioner: salt_version: latest From 1367b096cab5faa68e0e37421c41da1d06eaeaf1 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 18 Apr 2023 18:10:18 -0700 Subject: [PATCH 38/57] couple places that 3006 is needed for git installs. --- .github/workflows/ci.yml | 24 ++++++++--------- .github/workflows/templates/generate.py | 34 ++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a649d7..5ea38be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -277,7 +277,7 @@ jobs: distro-slug: amazon-2 display-name: Amazon 2 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' arch: @@ -291,7 +291,7 @@ jobs: distro-slug: arch display-name: Arch timeout: 20 - instances: '["stable-3006", "git-3006", "git-master", "latest"]' + instances: '["git-master", "latest"]' centos-7: @@ -347,7 +347,7 @@ jobs: distro-slug: debian-10 display-name: Debian 10 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' debian-11: @@ -361,7 +361,7 @@ jobs: distro-slug: debian-11 display-name: Debian 11 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' fedora-35: @@ -375,7 +375,7 @@ jobs: distro-slug: fedora-35 display-name: Fedora 35 timeout: 20 - instances: '["git-3006", "git-master", "latest"]' + instances: '["git-master", "latest"]' fedora-36: @@ -389,7 +389,7 @@ jobs: distro-slug: fedora-36 display-name: Fedora 36 timeout: 20 - instances: '["git-3006", "git-master", "latest"]' + instances: '["git-master", "latest"]' fedora-37: @@ -431,7 +431,7 @@ jobs: distro-slug: gentoo display-name: Gentoo timeout: 90 - instances: '["stable-3006", "git-3006", "git-master", "latest"]' + instances: '["git-master", "latest"]' gentoo-systemd: @@ -445,7 +445,7 @@ jobs: distro-slug: gentoo-systemd display-name: Gentoo (systemd) timeout: 90 - instances: '["stable-3006", "git-3006", "git-master", "latest"]' + instances: '["git-master", "latest"]' opensuse-15: @@ -459,7 +459,7 @@ jobs: distro-slug: opensuse-15 display-name: Opensuse 15 timeout: 20 - instances: '["stable-3006", "latest"]' + instances: '["latest"]' opensuse-tumbleweed: @@ -473,7 +473,7 @@ jobs: distro-slug: opensuse-tumbleweed display-name: Opensuse Tumbleweed timeout: 20 - instances: '["stable-3006", "git-3006", "git-master", "latest"]' + instances: '["git-master", "latest"]' oraclelinux-7: @@ -543,7 +543,7 @@ jobs: distro-slug: ubuntu-2004 display-name: Ubuntu 20.04 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' ubuntu-2204: @@ -557,7 +557,7 @@ jobs: distro-slug: ubuntu-2204 display-name: Ubuntu 22.04 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' set-pipeline-exit-status: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 525eb7b..8274938 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -210,6 +210,36 @@ BLACKLIST_GIT_3005 = [ "ubuntu-2204", ] +BLACKLIST_3006 = [ + "arch", + "fedora-35", + "fedora-36", + "fedora-37", + "fedora-38", + "gentoo", + "gentoo-systemd", + "opensuse-15", + "opensuse-tumbleweed", +] + +BLACKLIST_GIT_3006 = [ + "amazon-2", + "arch", + "debian-10", + "debian-11", + "fedora-35", + "fedora-36", + "fedora-37", + "fedora-38", + "gentoo", + "gentoo-systemd", + "opensuse-15", + "opensuse-tumbleweed", + "ubuntu-2004", + "ubuntu-2110", + "ubuntu-2204", +] + SALT_VERSIONS = [ "3003", "3004", @@ -548,12 +578,14 @@ def generate_test_jobs(): "3003": BLACKLIST_3003, "3004": BLACKLIST_3004, "3005": BLACKLIST_3005, + "3006": BLACKLIST_3006, } if bootstrap_type == "git": BLACKLIST = { "3003": BLACKLIST_GIT_3003, "3004": BLACKLIST_GIT_3004, "3005": BLACKLIST_GIT_3005, + "3006": BLACKLIST_GIT_3006, } # .0 versions are a virtual version for pinning to the first @@ -563,7 +595,7 @@ def generate_test_jobs(): continue if ( - salt_version in ("3003", "3004", "3005") + salt_version in ("3003", "3004", "3005", "3006") and distro in BLACKLIST[salt_version] ): continue From e1fd4987bf74876b10a0ef865fe388ef737195c7 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 18 Apr 2023 18:50:26 -0700 Subject: [PATCH 39/57] updating kitchen.yml with 3006. --- kitchen.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/kitchen.yml b/kitchen.yml index 8ff0c11..9a74dfc 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -180,6 +180,20 @@ suites: - freebsd-131 - freebsd-123 - openbsd-6 + - name: git-3006 + provisioner: + salt_version: 3006 + salt_bootstrap_options: -x python3 -MPfq git %s + excludes: + - opensuse-15 + - opensuse-tumbleweed + - debian-11 + - arch + - gentoo + - gentoo-systemd + - freebsd-131 + - freebsd-123 + - openbsd-6 - name: stable-3003-0 provisioner: salt_version: 3003 @@ -246,6 +260,17 @@ suites: - freebsd-131 - freebsd-123 - openbsd-6 + - name: stable-3006 + provisioner: + salt_version: 3006 + salt_bootstrap_options: -x python3 -MP stable %s + excludes: + - opensuse-15 + - opensuse-tumbleweed + - arch + - freebsd-131 + - freebsd-123 + - openbsd-6 - name: git-master provisioner: salt_version: master @@ -271,6 +296,11 @@ suites: salt_version: 3005 salt_bootstrap_options: -MP onedir %s + - name: onedir-3006 + provisioner: + salt_version: 3006 + salt_bootstrap_options: -MP onedir %s + - name: onedir-rc-3006-0rc2 provisioner: salt_version: 3006.0rc2 From d8a7e90953368affa335b6837c34d67ecbc27a2c Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 18 Apr 2023 19:51:33 -0700 Subject: [PATCH 40/57] fixing 3006 tag name --- .github/workflows/templates/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 8274938..60854e4 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -263,7 +263,7 @@ VERSION_DISPLAY_NAMES = { "3003": "v3003", "3004": "v3004", "3005": "v3005", - "3006": "v3006", + "3006": "v3006.0", "master": "Master", "latest": "Latest", "nightly": "Nightly", From 6dbae5e91615059301cd3ab05fbf97e0a3a5bd03 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 18 Apr 2023 22:07:14 -0700 Subject: [PATCH 41/57] fixing version for git-3006. --- kitchen.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kitchen.yml b/kitchen.yml index 9a74dfc..4ae8923 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -182,7 +182,7 @@ suites: - openbsd-6 - name: git-3006 provisioner: - salt_version: 3006 + salt_version: 3006.0 salt_bootstrap_options: -x python3 -MPfq git %s excludes: - opensuse-15 From 142eb803051b2ecc97cfb9b00802a81a62f2a026 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 19 Apr 2023 09:25:08 -0700 Subject: [PATCH 42/57] removing check for whether to use the new GPG key. --- bootstrap-salt.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 87a83ee..cef9ae5 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -6351,16 +6351,10 @@ install_amazon_linux_ami_2_onedir_deps() { if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005)')" != "" ]; then gpg_key="${base_url}SALTSTACK-GPG-KEY.pub,${base_url}base/RPM-GPG-KEY-CentOS-7" if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then - gpg_key="${base_url}SALTSTACK-GPG-KEY.pub" - fi - elif [ "$(echo "${ONEDIR_REV}" | grep -E '(latest|nightly)')" != "" ]; then - if __check_url_exists "${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub"; then - gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub" - else gpg_key="${base_url}SALTSTACK-GPG-KEY.pub" fi else - gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub" + gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub" fi # This should prob be refactored to use __install_saltstack_rhel_repository() From f823932131280231aa6349a384002406648de93d Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 19 Apr 2023 12:39:29 -0700 Subject: [PATCH 43/57] Some fixes for MacOS. --- bootstrap-salt.sh | 20 ++++++++++++-------- kitchen.macos.yml | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index cef9ae5..3360943 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -628,7 +628,7 @@ elif [ "$ITYPE" = "stable" ]; then elif [ "$(echo "$1" | grep -E '^(3003|3004|3005)$')" != "" ]; then STABLE_REV="$1" shift - elif [ "$(echo "$1" | grep -E '^([3-9][6-9]{3}(\.[0-9]*)?)')" != "" ]; then + elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then ONEDIR_REV="minor/$1" _ONEDIR_REV="$1" ITYPE="onedir" @@ -8465,16 +8465,16 @@ __parse_repo_json_python() { # Using latest, grab the right # version from the repo.json - _JSON_LATEST_VERSION=$(python - <<-EOF + _JSON_VERSION=$(python - <<-EOF import json, urllib.request url = "https://repo.saltproject.io/salt/py3/macos/repo.json" response = urllib.request.urlopen(url) data = json.loads(response.read()) -version = data['latest'][list(data['latest'])[0]]['version'] +version = data["${_ONEDIR_REV}"][list(data["${_ONEDIR_REV}"])[0]]['version'] print(version) EOF ) -echo "${_JSON_LATEST_VERSION}" +echo "${_JSON_VERSION}" } __macosx_get_packagesite_onedir() { @@ -8486,14 +8486,18 @@ __macosx_get_packagesite_onedir() { fi if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then - _ONEDIR_REV=$(__parse_repo_json_python) + _PKG_VERSION=$(__parse_repo_json_python) + elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*))')" != "" ]; then + _PKG_VERSION=$_ONEDIR_REV + else + _PKG_VERSION=$(__parse_repo_json_python) fi if [ "$(echo "$_ONEDIR_REV" | grep -E '^(3005)')" != "" ]; then - PKG="salt-${_ONEDIR_REV}-macos-${DARWIN_ARCH}.pkg" + PKG="salt-${_PKG_VERSION}-macos-${DARWIN_ARCH}.pkg" else - PKG="salt-${_ONEDIR_REV}-${__PY_VERSION_REPO}-${DARWIN_ARCH}.pkg" + PKG="salt-${_PKG_VERSION}-${__PY_VERSION_REPO}-${DARWIN_ARCH}.pkg" fi - SALTPKGCONFURL="https://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/macos/minor/${_ONEDIR_REV}/${PKG}" + SALTPKGCONFURL="https://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/macos/${ONEDIR_REV}/${PKG}" } # Using a separate conf step to head for idempotent install... diff --git a/kitchen.macos.yml b/kitchen.macos.yml index d74b5fd..e4b0109 100644 --- a/kitchen.macos.yml +++ b/kitchen.macos.yml @@ -30,7 +30,7 @@ suites: salt_version: 3005.1 - name: stable-3006 provisioner: - salt_version: 3006.0 + salt_version: 3006 - name: latest provisioner: salt_version: latest From a3fed785f8d4502567f7dd70048b1562b1bfdfce Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 19 Apr 2023 13:17:46 -0700 Subject: [PATCH 44/57] More MacOS fixes. --- bootstrap-salt.sh | 11 +++++++++-- kitchen.macos.yml | 6 +++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 3360943..8a4d1d5 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -8627,8 +8627,15 @@ install_macosx_git_post() { install_macosx_restart_daemons() { [ $_START_DAEMONS -eq $BS_FALSE ] && return - /bin/launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist || return 1 - /bin/launchctl load -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist || return 1 + if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then + /bin/launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist || return 1 + /bin/launchctl load -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist || return 1 + fi + + if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then + /bin/launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.master.plist || return 1 + /bin/launchctl load -w /Library/LaunchDaemons/com.saltstack.salt.master.plist || return 1 + fi return 0 } diff --git a/kitchen.macos.yml b/kitchen.macos.yml index e4b0109..f6cd2dc 100644 --- a/kitchen.macos.yml +++ b/kitchen.macos.yml @@ -5,7 +5,6 @@ driver: provisioner: sudo: true salt_bootstrap_options: -MP stable %s - salt_call_command: /opt/salt/bin/salt-call init_environment: | echo 'auto_accept: true' > /tmp/auto-accept-keys.conf sudo mkdir -p /etc/salt/master.d @@ -22,18 +21,23 @@ suites: - name: stable-3003 provisioner: salt_version: 3003.4 + salt_call_command: /opt/salt/bin/salt-call - name: stable-3004 provisioner: salt_version: 3004.1 + salt_call_command: /opt/salt/bin/salt-call - name: stable-3005 provisioner: salt_version: 3005.1 + salt_call_command: /opt/salt/bin/salt-call - name: stable-3006 provisioner: salt_version: 3006 + salt_call_command: /opt/salt/salt-call - name: latest provisioner: salt_version: latest + salt_call_command: /opt/salt/salt-call verifier: command: pytest --cache-clear -v -s -ra --log-cli-level=debug -k "not test_ping" tests/integration/ From a8b32ab040f93d1d5bd626f16e09e8131f1ce98c Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 20 Apr 2023 11:39:52 -0700 Subject: [PATCH 45/57] Fixing bootstrap and CICD to install via tag only if version begins with v and use branch otherwise. --- .github/workflows/ci.yml | 6 +++--- .github/workflows/templates/generate.py | 16 ++++++++++++++-- bootstrap-salt.sh | 8 -------- kitchen.yml | 8 ++++---- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ea38be..7be637d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -263,7 +263,7 @@ jobs: distro-slug: almalinux-9 display-name: AlmaLinux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "onedir-nightly"]' + instances: '["git-v3005", "onedir-3005", "git-3006.x", "onedir-3006", "git-master", "onedir-nightly"]' amazon-2: @@ -333,7 +333,7 @@ jobs: distro-slug: centos-stream9 display-name: CentOS Stream 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "onedir-nightly"]' + instances: '["git-v3005", "onedir-3005", "git-3006.x", "onedir-3006", "git-master", "onedir-nightly"]' debian-10: @@ -529,7 +529,7 @@ jobs: distro-slug: rockylinux-9 display-name: Rocky Linux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "onedir-nightly"]' + instances: '["git-v3005", "onedir-3005", "git-3006.x", "onedir-3006", "git-master", "onedir-nightly"]' ubuntu-2004: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 60854e4..47eef3c 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -269,6 +269,14 @@ VERSION_DISPLAY_NAMES = { "nightly": "Nightly", } +GIT_VERSIONS = { + "3003": "v3003", + "3004": "v3004", + "3005": "v3005", + "3006": "3006.x", + "master": "master", +} + STABLE_VERSION_BLACKLIST = [ "master", "nightly", @@ -400,6 +408,7 @@ def generate_test_jobs(): # BSD's don't have a stable release, only use git for bootstrap_type in ("git",): + _salt_version = salt_version BLACKLIST = { "3003": BLACKLIST_3003, "3004": BLACKLIST_3004, @@ -415,6 +424,7 @@ def generate_test_jobs(): # there is no git version. if salt_version.endswith("-0"): continue + _salt_version = GIT_VERSIONS[salt_version] if ( salt_version in ("3003", "3004") @@ -422,7 +432,7 @@ def generate_test_jobs(): ): continue - kitchen_target = f"{bootstrap_type}-{salt_version}" + kitchen_target = f"{bootstrap_type}-{_salt_version}" instances.append(kitchen_target) if instances: @@ -546,6 +556,7 @@ def generate_test_jobs(): continue for bootstrap_type in ("stable", "git", "onedir", "onedir-rc"): + _salt_version = salt_version if bootstrap_type == "onedir": if salt_version not in ONEDIR_SALT_VERSIONS: continue @@ -593,6 +604,7 @@ def generate_test_jobs(): # there is no git version. if salt_version.endswith("-0"): continue + _salt_version = GIT_VERSIONS[salt_version] if ( salt_version in ("3003", "3004", "3005", "3006") @@ -600,7 +612,7 @@ def generate_test_jobs(): ): continue - kitchen_target = f"{bootstrap_type}-{salt_version}" + kitchen_target = f"{bootstrap_type}-{_salt_version}" instances.append(kitchen_target) if instances: diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 8a4d1d5..d392805 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -1961,10 +1961,6 @@ if [ "$ITYPE" = "git" ]; then if [ "$__NEW_VS_TAG_REGEX_MATCH" = "MATCH" ]; then _POST_NEON_INSTALL=$BS_TRUE __TAG_REGEX_MATCH="${__NEW_VS_TAG_REGEX_MATCH}" - if [ "$(echo "${GIT_REV}" | cut -c -1)" != "v" ]; then - # We do this to properly clone tags - GIT_REV="v${GIT_REV}" - fi echodebug "Post Neon Tag Regex Match On: ${GIT_REV}" else __TAG_REGEX_MATCH=$(echo "${GIT_REV}" | sed -E 's/^(v?[0-9]{1,4}\.[0-9]{1,2})(\.[0-9]{1,2})?.*$/MATCH/') @@ -1976,10 +1972,6 @@ if [ "$ITYPE" = "git" ]; then if [ "$__NEW_VS_TAG_REGEX_MATCH" = "MATCH" ]; then _POST_NEON_INSTALL=$BS_TRUE __TAG_REGEX_MATCH="${__NEW_VS_TAG_REGEX_MATCH}" - if [ "$(echo "${GIT_REV}" | cut -c -1)" != "v" ]; then - # We do this to properly clone tags - GIT_REV="v${GIT_REV}" - fi echodebug "Post Neon Tag Regex Match On: ${GIT_REV}" else __TAG_REGEX_MATCH=$(echo "${GIT_REV}" | sed 's/^.*\(v\?[[:digit:]]\{1,4\}\.[[:digit:]]\{1,2\}\)\(\.[[:digit:]]\{1,2\}\)\?.*$/MATCH/') diff --git a/kitchen.yml b/kitchen.yml index 4ae8923..8195070 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -150,11 +150,11 @@ platforms: suites: - name: git-3003 provisioner: - salt_version: 3003 + salt_version: v3003 salt_bootstrap_options: -x python3 -MPfq git %s - name: git-3004 provisioner: - salt_version: 3004 + salt_version: v3004 salt_bootstrap_options: -x python3 -MPfq git %s excludes: - opensuse-15 @@ -168,7 +168,7 @@ suites: - openbsd-6 - name: git-3005 provisioner: - salt_version: 3005 + salt_version: v3005 salt_bootstrap_options: -x python3 -MPfq git %s excludes: - opensuse-15 @@ -182,7 +182,7 @@ suites: - openbsd-6 - name: git-3006 provisioner: - salt_version: 3006.0 + salt_version: 3006.x salt_bootstrap_options: -x python3 -MPfq git %s excludes: - opensuse-15 From 719422adfe38cf727779502f4aba0609b8413ea9 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 20 Apr 2023 11:58:04 -0700 Subject: [PATCH 46/57] fixing kitchen.yml for git installs. --- kitchen.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kitchen.yml b/kitchen.yml index 8195070..630e4e7 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -148,11 +148,11 @@ platforms: run_command: /lib/systemd/systemd suites: - - name: git-3003 + - name: git-v3003 provisioner: salt_version: v3003 salt_bootstrap_options: -x python3 -MPfq git %s - - name: git-3004 + - name: git-v3004 provisioner: salt_version: v3004 salt_bootstrap_options: -x python3 -MPfq git %s @@ -166,7 +166,7 @@ suites: - freebsd-131 - freebsd-123 - openbsd-6 - - name: git-3005 + - name: git-v3005 provisioner: salt_version: v3005 salt_bootstrap_options: -x python3 -MPfq git %s @@ -180,7 +180,7 @@ suites: - freebsd-131 - freebsd-123 - openbsd-6 - - name: git-3006 + - name: git-3006.x provisioner: salt_version: 3006.x salt_bootstrap_options: -x python3 -MPfq git %s From e7c8f7698b83b9859d13d0692da9a052508ba203 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 20 Apr 2023 13:01:58 -0700 Subject: [PATCH 47/57] If the passed version begins with v, eg. it's a tag, remove it before comparison. --- tests/conftest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9e26a11..b24d90c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,9 @@ +import logging import os import pprint + import pytest import testinfra -import logging log = logging.getLogger(__name__) @@ -47,6 +48,8 @@ def target_python_version(): @pytest.fixture(scope="session") def target_salt_version(): target_salt = os.environ["KITCHEN_SUITE"].split("-", 2)[-1].replace("-", ".") + if target_salt.startswith("v"): + target_salt = target_salt[1:] if target_salt in ("latest", "master", "nightly"): pytest.skip("Don't have a specific salt version to test against") return target_salt From 15dfb8efdd4df13404208ace9a80b47432e27a98 Mon Sep 17 00:00:00 2001 From: Twangboy Date: Thu, 20 Apr 2023 14:31:28 -0600 Subject: [PATCH 48/57] Add support for 3006 on windows bootstrap script --- bootstrap-salt.ps1 | 445 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 344 insertions(+), 101 deletions(-) diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index 4943e8f..74c7b20 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -37,13 +37,13 @@ Specifies all the optional parameters in no particular order. .PARAMETER version - Default version defined in this script. + The version of the Salt minion to install. Default is "latest" which will + install the latest version of Salt minion available. .PARAMETER pythonVersion The version of Python the installer should use. Specify either "2" or "3". Beginning with Salt 2017.7.0, Salt will run on either Python 2 or Python 3. The default is Python 2 if not specified. This parameter only works for Salt - versions >= 2017.7.0. .PARAMETER runservice Boolean flag to start or stop the minion service. True will start the minion @@ -81,32 +81,47 @@ param( # Supports new version and latest # Option 1 means case insensitive [ValidatePattern('^(\d{4}(\.\d{1,2}){0,2}(\-\d{1})?)|(latest)$', Options=1)] - [string]$Version = '', + [Alias("v")] + [String]$Version = "latest", [Parameter(Mandatory=$false, ValueFromPipeline=$True)] - # Doesn't support Python versions prior to "2017.7.0" + # Python 3 support was added in 2017. Python 2 support was dropped in + # version 3001. This parameter is ignored for all versions before 2017 and + # after 3000. [ValidateSet("2","3")] - [string]$PythonVersion = "3", + [Alias("p")] + [String]$PythonVersion = "3", [Parameter(Mandatory=$false, ValueFromPipeline=$True)] [ValidateSet("true","false")] - [string]$RunService = "true", + [Alias("s")] + [String]$RunService = "true", [Parameter(Mandatory=$false, ValueFromPipeline=$True)] - [string]$Minion = "not-specified", + [Alias("m")] + [String]$Minion = "not-specified", [Parameter(Mandatory=$false, ValueFromPipeline=$True)] - [string]$Master = "not-specified", + [Alias("a")] + [String]$Master = "not-specified", [Parameter(Mandatory=$false, ValueFromPipeline=$True)] - [string]$RepoUrl= "https://repo.saltproject.io/windows", + [Alias("r")] + [String]$RepoUrl = "https://repo.saltproject.io/salt/py3/windows", [Parameter(Mandatory=$false, ValueFromPipeline=$True)] - [switch]$ConfigureOnly + [Alias("c")] + [Switch]$ConfigureOnly ) + +#=============================================================================== +# Script Preferences +#=============================================================================== # Powershell supports only TLS 1.0 by default. Add support for TLS 1.2 [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]'Tls12' +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" #=============================================================================== # Script Functions @@ -123,6 +138,129 @@ function Get-IsUacEnabled (Get-ItemProperty HKLM:\Software\Microsoft\Windows\CurrentVersion\Policies\System).EnableLua -ne 0 } +function Get-MajorVersion { + # Parses a version string and returns the major version + # + # Args: + # Version (string): The Version to parse + [CmdletBinding()] + param( + [Parameter(Mandatory=$true, Position=0)] + [String] $Version + ) + return ( $Version -split "\." )[0] +} + +function Convert-PSObjectToHashtable { + param ( + [Parameter(ValueFromPipeline)] + $InputObject + ) + if ($null -eq $InputObject) { return $null } + + $is_enum = $InputObject -is [System.Collections.IEnumerable] + $not_string = $InputObject -isnot [string] + if ($is_enum -and $not_string) { + $collection = @( + foreach ($object in $InputObject) { + Convert-PSObjectToHashtable $object + } + ) + + Write-Host -NoEnumerate $collection + } elseif ($InputObject -is [PSObject]) { + $hash = @{} + + foreach ($property in $InputObject.PSObject.Properties) { + $hash[$property.Name] = Convert-PSObjectToHashtable $property.Value + } + + $hash + } else { + $InputObject + } +} + +function Get-FileHash { + # Get-FileHash is a built-in cmdlet in powershell 5+ but we need to support + # powershell 3. This will overwrite the powershell 5 commandlet only for + # this script. But it will provide the missing cmdlet for powershell 3 + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [String] $Path, + + [Parameter(Mandatory=$false)] + [ValidateSet( + "SHA1", + "SHA256", + "SHA384", + "SHA512", + # https://serverfault.com/questions/820300/ + # why-isnt-mactripledes-algorithm-output-in-powershell-stable + "MACTripleDES", # don't use + "MD5", + "RIPEMD160", + IgnoreCase=$true)] + [String] $Algorithm = "SHA256" + ) + + if ( !(Test-Path $Path) ) { + Write-Verbose "Invalid path for hashing: $Path" + return @{} + } + + if ( (Get-Item -Path $Path) -isnot [System.IO.FileInfo]) { + Write-Verbose "Not a file for hashing: $Path" + return @{} + } + + $Path = Resolve-Path -Path $Path + + Switch ($Algorithm) { + SHA1 { + $hasher = [System.Security.Cryptography.SHA1CryptoServiceProvider]::Create() + } + SHA256 { + $hasher = [System.Security.Cryptography.SHA256]::Create() + } + SHA384 { + $hasher = [System.Security.Cryptography.SHA384]::Create() + } + SHA512 { + $hasher = [System.Security.Cryptography.SHA512]::Create() + } + MACTripleDES { + $hasher = [System.Security.Cryptography.MACTripleDES]::Create() + } + MD5 { + $hasher = [System.Security.Cryptography.MD5]::Create() + } + RIPEMD160 { + $hasher = [System.Security.Cryptography.RIPEMD160]::Create() + } + } + + Write-Verbose "Hashing using $Algorithm algorithm" + try { + $data = [System.IO.File]::OpenRead($Path) + $hash = $hasher.ComputeHash($data) + $hash = [System.BitConverter]::ToString($hash) -replace "-","" + return @{ + Path = $Path; + Algorithm = $Algorithm.ToUpper(); + Hash = $hash + } + } catch { + Write-Verbose "Error hashing: $Path" + return @{} + } finally { + if ($null -ne $data) { + $data.Close() + } + } +} + #=============================================================================== # Check for Elevated Privileges #=============================================================================== @@ -157,28 +295,56 @@ if (!(Get-IsAdministrator)) { } } +#=============================================================================== +# Change RepoUrl for older versions +#=============================================================================== +$defaultUrl = "https://repo.saltproject.io/salt/py3/windows" +$oldRepoUrl = "https://repo.saltproject.io/windows" +$majorVersion = Get-MajorVersion -Version $Version +$customUrl = $true +if ( $Version.ToLower() -ne "latest" ) { + # A specific version has been passed + # We only want to modify the URL if a custom URL was not passed + $uri = [Uri]($RepoUrl) + if ( $uri.AbsoluteUri -eq $defaultUrl ) { + # No customURL passed, let's check for a pre 3006 version + $customUrl = $false + if ( $majorVersion -lt "3006" ) { + # This is an older version, use the old URL + $RepoUrl = $oldRepoUrl + } else { + # This is a new URL, and a version was passed, let's look in minor + if ( $Version.ToLower() -ne $majorVersion.ToLower() ) { + $RepoUrl = "$RepoUrl/minor" + } + } + } +} else { + if ( $RepoUrl -eq $defaultUrl ) { + $customUrl = $false + } +} + #=============================================================================== # Verify Parameters #=============================================================================== Write-Verbose "Parameters passed in:" -Write-Verbose "version: $version" -Write-Verbose "runservice: $runservice" -Write-Verbose "master: $master" -Write-Verbose "minion: $minion" -Write-Verbose "repourl: $repourl" +Write-Verbose "version: $Version" +Write-Verbose "runservice: $RunService" +Write-Verbose "master: $Master" +Write-Verbose "minion: $Minion" +Write-Verbose "repourl: $RepoUrl" -if ($runservice.ToLower() -eq "true") { +if ($RunService.ToLower() -eq "true") { Write-Verbose "Windows service will be set to run" - [bool]$runservice = $True -} -elseif ($runservice.ToLower() -eq "false") { + [bool]$RunService = $True +} elseif ($RunService.ToLower() -eq "false") { Write-Verbose "Windows service will be stopped and set to manual" - [bool]$runservice = $False -} -else { + [bool]$RunService = $False +} else { # Param passed in wasn't clear so defaulting to true. Write-Verbose "Windows service defaulting to run automatically" - [bool]$runservice = $True + [bool]$RunService = $True } #=============================================================================== @@ -231,8 +397,14 @@ if (Test-Path C:\tmp\grains) { $ConfiguredAnything = $True } -if ($ConfigureOnly -and !$ConfiguredAnything) { - Write-Output "No configuration or keys were copied over. No configuration was done!" +if ( $ConfigureOnly ) { + if ( !$ConfiguredAnything ) { + Write-Host "No configuration or keys were copied over." -ForegroundColor yes + Write-Host "No configuration was done!" -ForegroundColor Yellow + } else { + Write-Host "Salt minion successfully configured" -ForegroundColor Green + } + # If we're only configuring, we want to end here exit 0 } @@ -246,97 +418,168 @@ if ([IntPtr]::Size -eq 4) { } #=============================================================================== -# Use version "Latest" if no version is passed +# Get file name to download #=============================================================================== -if ((!$version) -or ($version.ToLower() -eq 'latest')){ - $versionSection = "Latest-Py$PythonVersion" +$saltFileName = "" +$saltVersion = "" +$saltSha512= "" +$saltFileUrl = "" +if ( ($customUrl) -or ($majorVersion -lt 3006) ) { + $saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" + $saltVersion = $Version + $saltFileUrl = "$RepoUrl/$saltFileName" } else { - $versionSection = $version - $year = $version.Substring(0, 4) - if ([int]$year -ge 2017) { - if ($PythonVersion -eq "3") { - $versionSection = "$version-Py3" - } else { - $versionSection = "$version-Py2" + if ( $majorVersion -ge 3006 ) { + $enc = [System.Text.Encoding]::UTF8 + try { + $response = Invoke-WebRequest -Uri "$RepoUrl/repo.json" -UseBasicParsing + if ($response.Content.GetType().Name -eq "Byte[]") { + $psobj = $enc.GetString($response.Content) | ConvertFrom-Json + } else { + $psobj = $response.Content | ConvertFrom-Json + } + $hash = Convert-PSObjectToHashtable $psobj + } catch { + Write-Verbose "repo.json not found at: $RepoUrl" + $hash = @{} + } + + $searchVersion = $Version.ToLower() + if ( $hash.Contains($searchVersion)) { + foreach ($item in $hash.($searchVersion).Keys) { + if ( $item.EndsWith(".exe") ) { + if ( $item.Contains($arch) ) { + $saltFileName = $hash.($searchVersion).($item).name + $saltVersion = $hash.($searchVersion).($item).version + $saltSha512 = $hash.($searchVersion).($item).SHA512 + } + } + } + } + if ( $saltFileName -and $saltVersion -and $saltSha512 ) { + if ( $RepoUrl.Contains("minor") ) { + $saltFileUrl = @($RepoUrl, $saltVersion, $saltFileName) -join "/" + } else { + $saltFileUrl = @($RepoUrl, "minor", $saltVersion, $saltFileName) -join "/" + } } } } -if (!$ConfigureOnly) { - #=============================================================================== - # Download minion setup file - #=============================================================================== - $saltExe = "Salt-Minion-$versionSection-$arch-Setup.exe" - Write-Output "Downloading Salt minion installer $saltExe" - $webclient = New-Object System.Net.WebClient - $url = "$repourl/$saltExe" - $file = "C:\Windows\Temp\$saltExe" - $webclient.DownloadFile($url, $file) +#=============================================================================== +# Download minion setup file +#=============================================================================== +Write-Host "===============================================================================" -ForegroundColor Yellow +Write-Host " Bootstrapping Salt Minion" -ForegroundColor Green +Write-Host " - version: $Version" +Write-Host " - file name: $saltFileName" +Write-Host " - file url: $saltFileUrl" +Write-Host "-------------------------------------------------------------------------------" -ForegroundColor Yellow +Write-Host "Downloading Installer: " -NoNewline +$webclient = New-Object System.Net.WebClient +$localFile = "C:\Windows\Temp\$saltFileName" +$webclient.DownloadFile($saltFileUrl, $localFile) - #=============================================================================== - # Set the parameters for the installer - #=============================================================================== - # Unless specified, use the installer defaults - # - id: - # - master: salt - # - Start the service - $parameters = "" - if($minion -ne "not-specified") {$parameters = "/minion-name=$minion"} - if($master -ne "not-specified") {$parameters = "$parameters /master=$master"} - if($runservice -eq $false) {$parameters = "$parameters /start-service=0"} +if ( Test-Path -Path $localFile ) { + Write-Host "Success" -ForegroundColor Green +} else { + Write-Host "Failed" -ForegroundColor Red +} - #=============================================================================== - # Install minion silently - #=============================================================================== - #Wait for process to exit before continuing. - Write-Output "Installing Salt minion" - Start-Process C:\Windows\Temp\$saltExe -ArgumentList "/S $parameters" -Wait -NoNewWindow -PassThru | Out-Null - - #=============================================================================== - # Configure the minion service - #=============================================================================== - # Wait for salt-minion service to be registered before trying to start it - $service = Get-Service salt-minion -ErrorAction SilentlyContinue - while (!$service) { - Start-Sleep -s 2 - $service = Get-Service salt-minion -ErrorAction SilentlyContinue +if ( $saltSha512 ) { + $localSha512 = (Get-FileHash -Path $localFile -Algorithm SHA512).Hash + Write-Host "Comparing Hash: " -NoNewline + if ( $localSha512 -eq $saltSha512 ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 } +} - if($runservice) { - # Start service - Write-Output "Starting the Salt minion service" +#=============================================================================== +# Set the parameters for the installer +#=============================================================================== +# Unless specified, use the installer defaults +# - id: +# - master: salt +# - Start the service +$parameters = "" +if($Minion -ne "not-specified") {$parameters = "/minion-name=$Minion"} +if($Master -ne "not-specified") {$parameters = "$parameters /master=$Master"} +if($RunService -eq $false) {$parameters = "$parameters /start-service=0"} + +#=============================================================================== +# Install minion silently +#=============================================================================== +#Wait for process to exit before continuing. +Write-Host "Installing Salt Minion: " -NoNewline +Start-Process $localFile -ArgumentList "/S $parameters" -Wait -NoNewWindow -PassThru | Out-Null + +#=============================================================================== +# Configure the minion service +#=============================================================================== +# Wait for salt-minion service to be registered before trying to start it +$service = Get-Service salt-minion -ErrorAction SilentlyContinue +while (!$service) { + Start-Sleep -s 2 + $service = Get-Service salt-minion -ErrorAction SilentlyContinue +} +if ( $service ) { + Write-Host "Success" -ForegroundColor Green +} else { + Write-Host "Failed" -ForegroundColor Red + exit 1 +} + +if($RunService) { + # Start service + Write-Host "Starting Service: " -NoNewline + Start-Service -Name "salt-minion" -ErrorAction SilentlyContinue + + # Check if service is started, otherwise retry starting the + # service 4 times. + $try = 0 + while (($service.Status -ne "Running") -and ($try -ne 4)) { Start-Service -Name "salt-minion" -ErrorAction SilentlyContinue - - # Check if service is started, otherwise retry starting the - # service 4 times. - $try = 0 - while (($service.Status -ne "Running") -and ($try -ne 4)) { - Start-Service -Name "salt-minion" -ErrorAction SilentlyContinue - $service = Get-Service salt-minion -ErrorAction SilentlyContinue - Start-Sleep -s 2 - $try += 1 - } - - # If the salt-minion service is still not running, something probably - # went wrong and user intervention is required - report failure. - if ($service.Status -eq "Stopped") { - Write-Output -NoNewline "Failed to start salt minion" - exit 1 - } + $service = Get-Service salt-minion -ErrorAction SilentlyContinue + Start-Sleep -s 2 + $try += 1 } - else { - Write-Output -NoNewline "Stopping salt minion and setting it to 'Manual'" - Set-Service "salt-minion" -StartupType "Manual" - Stop-Service "salt-minion" + + # If the salt-minion service is still not running, something probably + # went wrong and user intervention is required - report failure. + if ($service.Status -eq "Running") { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } + +} else { + Write-Host "Setting Service to 'Manual': " -NoNewline + Set-Service "salt-minion" -StartupType "Manual" + if ( (Get-Service "salt-minion").StartType -eq "Manual" ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } + + Write-Host "Stopping Service: " -NoNewline + Stop-Service "salt-minion" + if ( (Get-Service "salt-minion").Status -eq "Stopped" ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 } } #=============================================================================== # Script Complete #=============================================================================== -if ($ConfigureOnly) { - Write-Output "Salt minion successfully configured" -} -else { - Write-Output "Salt minion successfully installed" -} +Write-Host "-------------------------------------------------------------------------------" -ForegroundColor Yellow +Write-Host "Salt Minion Installed Successfully" -ForegroundColor Green +Write-Host "===============================================================================" -ForegroundColor Yellow +exit 0 From c2c6bd1e701758f59b0b63f7cbf5818b30ab9a60 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 20 Apr 2023 14:23:12 -0700 Subject: [PATCH 49/57] revert GIT_VERSIONS code. --- .github/workflows/ci.yml | 6 +++--- .github/workflows/templates/generate.py | 16 ++-------------- kitchen.yml | 2 +- 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7be637d..5ea38be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -263,7 +263,7 @@ jobs: distro-slug: almalinux-9 display-name: AlmaLinux 9 timeout: 20 - instances: '["git-v3005", "onedir-3005", "git-3006.x", "onedir-3006", "git-master", "onedir-nightly"]' + instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "onedir-nightly"]' amazon-2: @@ -333,7 +333,7 @@ jobs: distro-slug: centos-stream9 display-name: CentOS Stream 9 timeout: 20 - instances: '["git-v3005", "onedir-3005", "git-3006.x", "onedir-3006", "git-master", "onedir-nightly"]' + instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "onedir-nightly"]' debian-10: @@ -529,7 +529,7 @@ jobs: distro-slug: rockylinux-9 display-name: Rocky Linux 9 timeout: 20 - instances: '["git-v3005", "onedir-3005", "git-3006.x", "onedir-3006", "git-master", "onedir-nightly"]' + instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "onedir-nightly"]' ubuntu-2004: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 47eef3c..60854e4 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -269,14 +269,6 @@ VERSION_DISPLAY_NAMES = { "nightly": "Nightly", } -GIT_VERSIONS = { - "3003": "v3003", - "3004": "v3004", - "3005": "v3005", - "3006": "3006.x", - "master": "master", -} - STABLE_VERSION_BLACKLIST = [ "master", "nightly", @@ -408,7 +400,6 @@ def generate_test_jobs(): # BSD's don't have a stable release, only use git for bootstrap_type in ("git",): - _salt_version = salt_version BLACKLIST = { "3003": BLACKLIST_3003, "3004": BLACKLIST_3004, @@ -424,7 +415,6 @@ def generate_test_jobs(): # there is no git version. if salt_version.endswith("-0"): continue - _salt_version = GIT_VERSIONS[salt_version] if ( salt_version in ("3003", "3004") @@ -432,7 +422,7 @@ def generate_test_jobs(): ): continue - kitchen_target = f"{bootstrap_type}-{_salt_version}" + kitchen_target = f"{bootstrap_type}-{salt_version}" instances.append(kitchen_target) if instances: @@ -556,7 +546,6 @@ def generate_test_jobs(): continue for bootstrap_type in ("stable", "git", "onedir", "onedir-rc"): - _salt_version = salt_version if bootstrap_type == "onedir": if salt_version not in ONEDIR_SALT_VERSIONS: continue @@ -604,7 +593,6 @@ def generate_test_jobs(): # there is no git version. if salt_version.endswith("-0"): continue - _salt_version = GIT_VERSIONS[salt_version] if ( salt_version in ("3003", "3004", "3005", "3006") @@ -612,7 +600,7 @@ def generate_test_jobs(): ): continue - kitchen_target = f"{bootstrap_type}-{_salt_version}" + kitchen_target = f"{bootstrap_type}-{salt_version}" instances.append(kitchen_target) if instances: diff --git a/kitchen.yml b/kitchen.yml index 630e4e7..79da273 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -180,7 +180,7 @@ suites: - freebsd-131 - freebsd-123 - openbsd-6 - - name: git-3006.x + - name: git-3006 provisioner: salt_version: 3006.x salt_bootstrap_options: -x python3 -MPfq git %s From d9a8f4a6d8d4672a7af48fce1acd09078b310b77 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 20 Apr 2023 14:40:02 -0700 Subject: [PATCH 50/57] Fixing kitchen.yml file. --- kitchen.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kitchen.yml b/kitchen.yml index 79da273..8195070 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -148,11 +148,11 @@ platforms: run_command: /lib/systemd/systemd suites: - - name: git-v3003 + - name: git-3003 provisioner: salt_version: v3003 salt_bootstrap_options: -x python3 -MPfq git %s - - name: git-v3004 + - name: git-3004 provisioner: salt_version: v3004 salt_bootstrap_options: -x python3 -MPfq git %s @@ -166,7 +166,7 @@ suites: - freebsd-131 - freebsd-123 - openbsd-6 - - name: git-v3005 + - name: git-3005 provisioner: salt_version: v3005 salt_bootstrap_options: -x python3 -MPfq git %s From fe0c97555596d85650852df3804f938a28dc7433 Mon Sep 17 00:00:00 2001 From: Twangboy Date: Thu, 20 Apr 2023 16:02:53 -0600 Subject: [PATCH 51/57] Add 3006 to the tests, fix salt_call_command --- .github/workflows/ci.yml | 4 ++-- kitchen.windows.yml | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e01c95..3e0f693 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -219,7 +219,7 @@ jobs: display-name: Windows 2019 timeout: 20 runs-on: windows-2019 - instances: '["stable-3003", "stable-3004", "stable-3005", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "stable-3006", "latest"]' windows-2022: @@ -234,7 +234,7 @@ jobs: display-name: Windows 2022 timeout: 20 runs-on: windows-2022 - instances: '["stable-3003", "stable-3004", "stable-3005", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "stable-3006", "latest"]' diff --git a/kitchen.windows.yml b/kitchen.windows.yml index f68f4b1..ccd3f05 100644 --- a/kitchen.windows.yml +++ b/kitchen.windows.yml @@ -20,16 +20,23 @@ suites: - name: stable-3003 provisioner: salt_version: 3003.4-1 + salt_call_command: c:\salt\salt-call.bat - name: stable-3004 provisioner: salt_version: 3004.1-1 + salt_call_command: c:\Program Files\Salt Project\Salt\salt-call.bat - name: stable-3005 provisioner: salt_version: 3005.1-1 + salt_call_command: c:\Program Files\Salt Project\Salt\salt-call.bat + - name: stable-3006 + provisioner: + salt_version: 3006.0 + salt_call_command: c:\Program Files\Salt Project\Salt\salt-call.exe - name: latest provisioner: salt_version: latest - salt_call_command: c:\Program Files\Salt Project\Salt\salt-call.bat + salt_call_command: c:\Program Files\Salt Project\Salt\salt-call.exe verifier: command: pytest --cache-clear -v -s -ra --log-cli-level=debug -k "not test_ping" tests/integration/ From 8d02826be71a1b5852c946da089d1bd1ad05b895 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 20 Apr 2023 15:30:38 -0700 Subject: [PATCH 52/57] account for salt_version ending in .x, eg. 3006.x branch. --- tests/integration/test_installation.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index aacc7de..9ee18d4 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -1,6 +1,5 @@ -# -*- coding: utf-8 -*- -import os import logging +import os from contextlib import nullcontext log = logging.getLogger(__name__) @@ -26,7 +25,7 @@ def test_target_python_version(host, target_python_version): def test_target_salt_version(host, target_salt_version): with selected_context_manager(host): ret = host.salt("grains.item", "saltversion", "--timeout=120") - if target_salt_version.endswith(".0"): + if target_salt_version.endswith(".0") or target_salt_version.endswith(".x"): assert ret["saltversion"] == ".".join(target_salt_version.split(".")[:-1]) else: assert ret["saltversion"].startswith(target_salt_version) From 5e6bcb58153f8d3d5e7a54db4d22dc31f1b5daf9 Mon Sep 17 00:00:00 2001 From: Twangboy Date: Thu, 20 Apr 2023 16:44:17 -0600 Subject: [PATCH 53/57] Fix the 3006 tests --- .github/workflows/ci.yml | 48 ++++++++++++------------- .github/workflows/templates/generate.py | 16 ++++----- .gitignore | 3 ++ 3 files changed, 33 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e0f693..c32c082 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,7 +173,7 @@ jobs: display-name: macOS 10.15 timeout: 20 runs-on: macos-10.15 - instances: '["stable-3003", "stable-3004", "stable-3005", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "stable-3006", "latest"]' macos-11: @@ -188,7 +188,7 @@ jobs: display-name: macOS 11 timeout: 20 runs-on: macos-11 - instances: '["stable-3003", "stable-3004", "stable-3005", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "stable-3006", "latest"]' macos-12: @@ -203,7 +203,7 @@ jobs: display-name: macOS 12 timeout: 20 runs-on: macos-12 - instances: '["stable-3003", "stable-3004", "stable-3005", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "stable-3006", "latest"]' @@ -249,7 +249,7 @@ jobs: distro-slug: almalinux-8 display-name: AlmaLinux 8 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "latest", "onedir-nightly"]' almalinux-9: @@ -263,7 +263,7 @@ jobs: distro-slug: almalinux-9 display-name: AlmaLinux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-master", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["git-3005", "onedir-3005", "git-3006", "git-master", "onedir-nightly"]' amazon-2: @@ -277,7 +277,7 @@ jobs: distro-slug: amazon-2 display-name: Amazon 2 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "git-master", "latest", "onedir-nightly"]' arch: @@ -291,7 +291,7 @@ jobs: distro-slug: arch display-name: Arch timeout: 20 - instances: '["git-master", "latest"]' + instances: '["stable-3006", "git-3006", "git-master", "latest"]' centos-7: @@ -305,7 +305,7 @@ jobs: distro-slug: centos-7 display-name: CentOS 7 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "latest", "onedir-nightly"]' centos-stream8: @@ -319,7 +319,7 @@ jobs: distro-slug: centos-stream8 display-name: CentOS Stream 8 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "latest", "onedir-nightly"]' centos-stream9: @@ -333,7 +333,7 @@ jobs: distro-slug: centos-stream9 display-name: CentOS Stream 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-master", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["git-3005", "onedir-3005", "git-3006", "git-master", "onedir-nightly"]' debian-10: @@ -347,7 +347,7 @@ jobs: distro-slug: debian-10 display-name: Debian 10 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "git-master", "latest", "onedir-nightly"]' debian-11: @@ -361,7 +361,7 @@ jobs: distro-slug: debian-11 display-name: Debian 11 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "git-master", "latest", "onedir-nightly"]' fedora-35: @@ -375,7 +375,7 @@ jobs: distro-slug: fedora-35 display-name: Fedora 35 timeout: 20 - instances: '["git-master", "latest"]' + instances: '["git-3006", "git-master", "latest"]' fedora-36: @@ -389,7 +389,7 @@ jobs: distro-slug: fedora-36 display-name: Fedora 36 timeout: 20 - instances: '["git-master", "latest"]' + instances: '["git-3006", "git-master", "latest"]' fedora-37: @@ -431,7 +431,7 @@ jobs: distro-slug: gentoo display-name: Gentoo timeout: 90 - instances: '["git-master", "latest"]' + instances: '["stable-3006", "git-3006", "git-master", "latest"]' gentoo-systemd: @@ -445,7 +445,7 @@ jobs: distro-slug: gentoo-systemd display-name: Gentoo (systemd) timeout: 90 - instances: '["git-master", "latest"]' + instances: '["stable-3006", "git-3006", "git-master", "latest"]' opensuse-15: @@ -459,7 +459,7 @@ jobs: distro-slug: opensuse-15 display-name: Opensuse 15 timeout: 20 - instances: '["latest"]' + instances: '["stable-3006", "latest"]' opensuse-tumbleweed: @@ -473,7 +473,7 @@ jobs: distro-slug: opensuse-tumbleweed display-name: Opensuse Tumbleweed timeout: 20 - instances: '["git-master", "latest"]' + instances: '["stable-3006", "git-3006", "git-master", "latest"]' oraclelinux-7: @@ -487,7 +487,7 @@ jobs: distro-slug: oraclelinux-7 display-name: Oracle Linux 7 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "latest", "onedir-nightly"]' oraclelinux-8: @@ -501,7 +501,7 @@ jobs: distro-slug: oraclelinux-8 display-name: Oracle Linux 8 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "latest", "onedir-nightly"]' rockylinux-8: @@ -515,7 +515,7 @@ jobs: distro-slug: rockylinux-8 display-name: Rocky Linux 8 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "latest", "onedir-nightly"]' rockylinux-9: @@ -529,7 +529,7 @@ jobs: distro-slug: rockylinux-9 display-name: Rocky Linux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-master", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["git-3005", "onedir-3005", "git-3006", "git-master", "onedir-nightly"]' ubuntu-2004: @@ -543,7 +543,7 @@ jobs: distro-slug: ubuntu-2004 display-name: Ubuntu 20.04 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "git-master", "latest", "onedir-nightly"]' ubuntu-2204: @@ -557,7 +557,7 @@ jobs: distro-slug: ubuntu-2204 display-name: Ubuntu 22.04 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "git-master", "latest", "onedir-nightly"]' set-pipeline-exit-status: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index a966c17..3476f8b 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -1,9 +1,8 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- -import os -import json -import pathlib import datetime +import json +import os +import pathlib os.chdir(os.path.abspath(os.path.dirname(__file__))) @@ -215,11 +214,10 @@ SALT_VERSIONS = [ "3003", "3004", "3005", + "3006", "master", "latest", "nightly", - "3006-0rc1", - "3006-0rc2", ] ONEDIR_SALT_VERSIONS = [ @@ -228,15 +226,13 @@ ONEDIR_SALT_VERSIONS = [ "nightly", ] -ONEDIR_RC_SALT_VERSIONS = [ - "3006-0rc1", - "3006-0rc2", -] +ONEDIR_RC_SALT_VERSIONS = [] VERSION_DISPLAY_NAMES = { "3003": "v3003", "3004": "v3004", "3005": "v3005", + "3006": "v3006", "master": "Master", "latest": "Latest", "nightly": "Nightly", diff --git a/.gitignore b/.gitignore index 45c7a12..9b7ebc0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ /.project *.sw? +*.un~ .vagrant +tools +venv # Pycharm .idea From 7825da5e89162340e6bed2f128ee83d922f846b3 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 20 Apr 2023 17:23:30 -0700 Subject: [PATCH 54/57] Revert to installing 3006 from the tag but blacklist any distribution that was running it until 3006.1 is out. --- .github/workflows/ci.yml | 6 +++--- .github/workflows/templates/generate.py | 3 +++ kitchen.yml | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ea38be..d47eb6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -263,7 +263,7 @@ jobs: distro-slug: almalinux-9 display-name: AlmaLinux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "onedir-nightly"]' + instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master", "onedir-nightly"]' amazon-2: @@ -333,7 +333,7 @@ jobs: distro-slug: centos-stream9 display-name: CentOS Stream 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "onedir-nightly"]' + instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master", "onedir-nightly"]' debian-10: @@ -529,7 +529,7 @@ jobs: distro-slug: rockylinux-9 display-name: Rocky Linux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "onedir-nightly"]' + instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master", "onedir-nightly"]' ubuntu-2004: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 60854e4..b9a97ce 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -223,8 +223,10 @@ BLACKLIST_3006 = [ ] BLACKLIST_GIT_3006 = [ + "almalinux-9", "amazon-2", "arch", + "centos-stream9", "debian-10", "debian-11", "fedora-35", @@ -235,6 +237,7 @@ BLACKLIST_GIT_3006 = [ "gentoo-systemd", "opensuse-15", "opensuse-tumbleweed", + "rockylinux-9", "ubuntu-2004", "ubuntu-2110", "ubuntu-2204", diff --git a/kitchen.yml b/kitchen.yml index 8195070..615a8b2 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -182,7 +182,7 @@ suites: - openbsd-6 - name: git-3006 provisioner: - salt_version: 3006.x + salt_version: v3006 salt_bootstrap_options: -x python3 -MPfq git %s excludes: - opensuse-15 From 3c8b08c2d1dbdcb795bde0c38495eafd62483001 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 20 Apr 2023 17:24:49 -0700 Subject: [PATCH 55/57] removing nightly from onedir until S3 buckets are fixed. --- .github/workflows/ci.yml | 28 ++++++++++++------------- .github/workflows/templates/generate.py | 1 - 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d47eb6b..e81e9b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -249,7 +249,7 @@ jobs: distro-slug: almalinux-8 display-name: AlmaLinux 8 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest", "onedir-nightly"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest"]' almalinux-9: @@ -263,7 +263,7 @@ jobs: distro-slug: almalinux-9 display-name: AlmaLinux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master", "onedir-nightly"]' + instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master"]' amazon-2: @@ -277,7 +277,7 @@ jobs: distro-slug: amazon-2 display-name: Amazon 2 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' arch: @@ -305,7 +305,7 @@ jobs: distro-slug: centos-7 display-name: CentOS 7 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest", "onedir-nightly"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest"]' centos-stream8: @@ -319,7 +319,7 @@ jobs: distro-slug: centos-stream8 display-name: CentOS Stream 8 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest", "onedir-nightly"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest"]' centos-stream9: @@ -333,7 +333,7 @@ jobs: distro-slug: centos-stream9 display-name: CentOS Stream 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master", "onedir-nightly"]' + instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master"]' debian-10: @@ -347,7 +347,7 @@ jobs: distro-slug: debian-10 display-name: Debian 10 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' debian-11: @@ -361,7 +361,7 @@ jobs: distro-slug: debian-11 display-name: Debian 11 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' fedora-35: @@ -487,7 +487,7 @@ jobs: distro-slug: oraclelinux-7 display-name: Oracle Linux 7 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest", "onedir-nightly"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest"]' oraclelinux-8: @@ -501,7 +501,7 @@ jobs: distro-slug: oraclelinux-8 display-name: Oracle Linux 8 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest", "onedir-nightly"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest"]' rockylinux-8: @@ -515,7 +515,7 @@ jobs: distro-slug: rockylinux-8 display-name: Rocky Linux 8 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest", "onedir-nightly"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest"]' rockylinux-9: @@ -529,7 +529,7 @@ jobs: distro-slug: rockylinux-9 display-name: Rocky Linux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master", "onedir-nightly"]' + instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master"]' ubuntu-2004: @@ -543,7 +543,7 @@ jobs: distro-slug: ubuntu-2004 display-name: Ubuntu 20.04 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' ubuntu-2204: @@ -557,7 +557,7 @@ jobs: distro-slug: ubuntu-2204 display-name: Ubuntu 22.04 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest", "onedir-nightly"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' set-pipeline-exit-status: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index b9a97ce..dfa960b 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -257,7 +257,6 @@ ONEDIR_SALT_VERSIONS = [ "3005", "3006", "latest", - "nightly", ] ONEDIR_RC_SALT_VERSIONS = [] From 4183713e82bf21eb13493cbd69e6f884db46508f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 21 Apr 2023 16:59:12 +0100 Subject: [PATCH 56/57] Don't use GITHUB_OUTPUT (#1911) Signed-off-by: Pedro Algarvio --- .github/workflows/release.yml | 1 - .github/workflows/scripts/cut-release.py | 11 ++++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 05c8c43..fa9c126 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,7 +61,6 @@ jobs: needs: - check-requirements outputs: - release-changes: ${{ steps.update-repo.outputs.release-changes }} release-version: ${{ steps.update-repo.outputs.release-version }} steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/scripts/cut-release.py b/.github/workflows/scripts/cut-release.py index 569907c..3b6eabf 100644 --- a/.github/workflows/scripts/cut-release.py +++ b/.github/workflows/scripts/cut-release.py @@ -1,12 +1,13 @@ #!/usr/bin/env python +import argparse import os +import pathlib import re import sys -import pathlib -import argparse -import requests from datetime import datetime +import requests + REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent.parent @@ -175,10 +176,6 @@ def main(): flush=True, ) cut_release_changes.write_text(changelog["body"]) - if github_output is not None: - with open(github_output, "a", encoding="utf-8") as wfh: - wfh.write(f"release-changes={changelog['body']}\n") - print( f"* Updating {changelog_file.relative_to(REPO_ROOT)} ...", file=sys.stderr, From 8d0c7e4d5fe8d833940c34091a5714b07c4d5f7d Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Fri, 21 Apr 2023 16:56:31 +0000 Subject: [PATCH 57/57] Update develop branch for the v2023.04.21 release --- CHANGELOG.md | 12 ++++++++++++ bootstrap-salt.sh | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 277008e..63788aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# v2023.04.21 + +## What's Changed + +- Update release process by @s0undt3ch in https://github.com/saltstack/salt-bootstrap/pull/1908 +- Update with 3006 stable installs by @garethgreenaway in https://github.com/saltstack/salt-bootstrap/pull/1906 +- Add support for 3006 on windows bootstrap script by @twangboy in https://github.com/saltstack/salt-bootstrap/pull/1918 +- Fixes for 3006 by @garethgreenaway in https://github.com/saltstack/salt-bootstrap/pull/1912 +- Don't use GITHUB_OUTPUT by @s0undt3ch in https://github.com/saltstack/salt-bootstrap/pull/1911 + +**Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2023.04.06...v2023.04.21 + # v2023.04.06 ## What's Changed diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d392805..2b056a2 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -23,7 +23,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2023.04.06" +__ScriptVersion="2023.04.21" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0"