From be6b928ac8f14bf4ecc0026f035310673294a752 Mon Sep 17 00:00:00 2001 From: Imran Iqbal Date: Mon, 5 Dec 2022 08:06:03 +0000 Subject: [PATCH 1/9] feat(bootstrap-salt): add common `onedir nightly` adjustments --- bootstrap-salt.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 2f4bb6f..f079348 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -269,6 +269,7 @@ _CUSTOM_MINION_CONFIG="null" _QUIET_GIT_INSTALLATION=$BS_FALSE _REPO_URL="repo.saltproject.io" _ONEDIR_DIR="salt" +_ONEDIR_NIGHTLY_DIR="salt-dev/${_ONEDIR_DIR}" _PY_EXE="python3" _INSTALL_PY="$BS_FALSE" _TORNADO_MAX_PY3_VERSION="5.0" @@ -636,7 +637,7 @@ elif [ "$ITYPE" = "onedir" ]; then if [ "$#" -eq 0 ];then ONEDIR_REV="latest" else - if [ "$(echo "$1" | grep -E '^(latest|3005)$')" != "" ]; then + if [ "$(echo "$1" | grep -E '^(nightly|latest|3005)$')" != "" ]; then ONEDIR_REV="$1" shift elif [ "$(echo "$1" | grep -E '^(3005(\.[0-9]*)?)')" != "" ]; then @@ -648,7 +649,7 @@ elif [ "$ITYPE" = "onedir" ]; then ONEDIR_REV="minor/$1" shift else - echo "Unknown stable version: $1 (valid: 3005, latest.)" + echo "Unknown stable version: $1 (valid: 3005, latest, nightly.)" exit 1 fi fi From f9c561adb0bb77e0b29c668236a97ec93597bb92 Mon Sep 17 00:00:00 2001 From: Imran Iqbal Date: Mon, 5 Dec 2022 08:06:47 +0000 Subject: [PATCH 2/9] feat(bootstrap-salt): add Debian-specific `onedir nightly` --- bootstrap-salt.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index f079348..81a77a5 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3577,6 +3577,9 @@ __install_saltstack_debian_onedir_repository() { # amd64 is just a part of repository URI, 32-bit pkgs are hosted under the same location SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/${ONEDIR_REV}/" + if [ "${ONEDIR_REV}" = "nightly" ] ; then + SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/" + fi echo "$__REPO_ARCH_DEB $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/salt.list" __apt_key_fetch "${SALTSTACK_DEBIAN_URL}salt-archive-keyring.gpg" || return 1 From 4d55d1b26d8f2bbe695129066be3426a7434142f Mon Sep 17 00:00:00 2001 From: Imran Iqbal Date: Mon, 5 Dec 2022 08:11:23 +0000 Subject: [PATCH 3/9] feat(bootstrap-salt): add Ubuntu-specific `onedir nightly` --- bootstrap-salt.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 81a77a5..f797b47 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3055,6 +3055,9 @@ __install_saltstack_ubuntu_onedir_repository() { # SaltStack's stable Ubuntu repository: SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/${ONEDIR_REV}/" + if [ "${ONEDIR_REV}" = "nightly" ] ; then + SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/" + fi echo "$__REPO_ARCH_DEB $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/salt.list __apt_key_fetch "${SALTSTACK_UBUNTU_URL}salt-archive-keyring.gpg" || return 1 From 599ec89559819e197a8fd978f549e6c226393f05 Mon Sep 17 00:00:00 2001 From: Imran Iqbal Date: Mon, 5 Dec 2022 08:25:59 +0000 Subject: [PATCH 4/9] feat(bootstrap-salt): add RedHat-specific `onedir nightly` --- bootstrap-salt.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index f797b47..608bb68 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4445,6 +4445,9 @@ __install_saltstack_rhel_onedir_repository() { # Avoid using '$releasever' variable for yum. # Instead, this should work correctly on all RHEL variants. base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/\$basearch/${ONEDIR_REV}/" + if [ "${ONEDIR_REV}" = "nightly" ] ; then + base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/\$basearch/" + fi if [ "${DISTRO_MAJOR_VERSION}" -eq 9 ]; then gpg_key="SALTSTACK-GPG-KEY2.pub" else @@ -4471,6 +4474,9 @@ enabled_metadata=1 _eof fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}/" + if [ "${ONEDIR_REV}" = "nightly" ] ; then + fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" + fi for key in $gpg_key; do __rpm_import_gpg "${fetch_url}${key}" || return 1 done From 100d9727037f03b97ec04755d0c1362a593cf7e3 Mon Sep 17 00:00:00 2001 From: Imran Iqbal Date: Mon, 5 Dec 2022 08:57:41 +0000 Subject: [PATCH 5/9] feat(bootstrap-salt): add Amazon-specific `onedir nightly` --- bootstrap-salt.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 608bb68..625fa3e 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -6212,6 +6212,9 @@ install_amazon_linux_ami_2_onedir_deps() { fi base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/amazon/2/\$basearch/$repo_rev/" + if [ "${ONEDIR_REV}" = "nightly" ] ; then + base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/amazon/2/\$basearch/" + fi gpg_key="${base_url}SALTSTACK-GPG-KEY.pub,${base_url}base/RPM-GPG-KEY-CentOS-7" if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then gpg_key="${base_url}SALTSTACK-GPG-KEY.pub" From 83f9c1e190290bd03b8808f5d1764e83566556c0 Mon Sep 17 00:00:00 2001 From: Imran Iqbal Date: Mon, 5 Dec 2022 08:27:14 +0000 Subject: [PATCH 6/9] test(conftest): skip Salt version test for `nightly` --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5e9429d..370525b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,6 +46,6 @@ def target_python_version(): @pytest.fixture(scope="session") def target_salt_version(): target_salt = os.environ["KITCHEN_SUITE"].split("-", 2)[-1].replace("-", ".") - if target_salt in ("latest", "master"): + if target_salt in ("latest", "master", "nightly"): pytest.skip("Don't have a specific salt version to test against") return target_salt From 9b5017dd03d9c3b976c50930864f37ef2b8a76f0 Mon Sep 17 00:00:00 2001 From: Imran Iqbal Date: Mon, 5 Dec 2022 08:40:41 +0000 Subject: [PATCH 7/9] ci(kitchen): add `onedir-nightly` suite --- kitchen.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kitchen.yml b/kitchen.yml index acff15f..60a105a 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -221,6 +221,11 @@ suites: salt_version: latest salt_bootstrap_options: -MP stable %s + - name: onedir-nightly + provisioner: + salt_version: nightly + salt_bootstrap_options: -MP onedir %s + - name: onedir-latest provisioner: salt_version: latest From ad8d9667b255c7de6e42f7d274badf7857211e9b Mon Sep 17 00:00:00 2001 From: Imran Iqbal Date: Wed, 7 Dec 2022 06:52:47 +0000 Subject: [PATCH 8/9] refactor(generate): use `STABLE_VERSION_BLACKLIST` consistently --- .github/workflows/templates/generate.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 4180700..a92dfbc 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -169,7 +169,9 @@ VERSION_DISPLAY_NAMES = { "latest": "Latest", } -STABLE_VERSION_BLACKLIST = [] +STABLE_VERSION_BLACKLIST = [ + "master", +] LATEST_PKG_BLACKLIST = [] @@ -327,8 +329,7 @@ def generate_test_jobs(): for bootstrap_type in ("stable",): if bootstrap_type == "stable": - if salt_version == "master": - # For the master branch there's no stable build + if salt_version in STABLE_VERSION_BLACKLIST: continue kitchen_target = f"{bootstrap_type}-{salt_version}" @@ -368,8 +369,7 @@ def generate_test_jobs(): for bootstrap_type in ("stable",): if bootstrap_type == "stable": - if salt_version == "master": - # For the master branch there's no stable build + if salt_version in STABLE_VERSION_BLACKLIST: continue kitchen_target = f"{bootstrap_type}-{salt_version}" @@ -417,15 +417,11 @@ def generate_test_jobs(): continue if bootstrap_type == "stable": - if salt_version == "master": - # For the master branch there's no stable build + if salt_version in STABLE_VERSION_BLACKLIST: continue if distro not in STABLE_DISTROS: continue - if salt_version in STABLE_VERSION_BLACKLIST: - continue - if distro.startswith("fedora") and salt_version != "latest": # Fedora does not keep old builds around continue From 3c3dba8a6ddcb5889b80c46ec06abbb576b7344b Mon Sep 17 00:00:00 2001 From: Imran Iqbal Date: Wed, 7 Dec 2022 06:54:32 +0000 Subject: [PATCH 9/9] feat(generate): add `onedir-nightly` generation => CI --- .github/workflows/ci.yml | 24 ++++++++++---------- .github/workflows/templates/generate.py | 29 +++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57a4068..baa80b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -249,7 +249,7 @@ jobs: distro-slug: almalinux-8 display-name: AlmaLinux 8 timeout: 20 - instances: '["stable-3003", "git-3003", "stable-3004", "git-3004", "stable-3005", "git-3005", "onedir-3005", "git-master", "latest"]' + instances: '["stable-3003", "git-3003", "stable-3004", "git-3004", "stable-3005", "git-3005", "onedir-3005", "git-master", "latest", "onedir-nightly"]' amazon-2: @@ -263,7 +263,7 @@ jobs: distro-slug: amazon-2 display-name: Amazon 2 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "git-master", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly"]' arch: @@ -291,7 +291,7 @@ jobs: distro-slug: centos-7 display-name: CentOS 7 timeout: 20 - instances: '["stable-3003", "git-3003", "stable-3004", "git-3004", "stable-3005", "git-3005", "onedir-3005", "git-master", "latest"]' + instances: '["stable-3003", "git-3003", "stable-3004", "git-3004", "stable-3005", "git-3005", "onedir-3005", "git-master", "latest", "onedir-nightly"]' centos-stream8: @@ -305,7 +305,7 @@ jobs: distro-slug: centos-stream8 display-name: CentOS Stream 8 timeout: 20 - instances: '["stable-3003", "git-3003", "stable-3004", "git-3004", "stable-3005", "git-3005", "onedir-3005", "git-master", "latest"]' + instances: '["stable-3003", "git-3003", "stable-3004", "git-3004", "stable-3005", "git-3005", "onedir-3005", "git-master", "latest", "onedir-nightly"]' debian-10: @@ -319,7 +319,7 @@ jobs: distro-slug: debian-10 display-name: Debian 10 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "git-master", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly"]' debian-11: @@ -333,7 +333,7 @@ jobs: distro-slug: debian-11 display-name: Debian 11 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "git-master", "latest"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly"]' fedora-35: @@ -431,7 +431,7 @@ jobs: distro-slug: oraclelinux-7 display-name: Oracle Linux 7 timeout: 20 - instances: '["stable-3003", "git-3003", "stable-3004", "git-3004", "stable-3005", "git-3005", "onedir-3005", "git-master", "latest"]' + instances: '["stable-3003", "git-3003", "stable-3004", "git-3004", "stable-3005", "git-3005", "onedir-3005", "git-master", "latest", "onedir-nightly"]' oraclelinux-8: @@ -445,7 +445,7 @@ jobs: distro-slug: oraclelinux-8 display-name: Oracle Linux 8 timeout: 20 - instances: '["stable-3003", "git-3003", "stable-3004", "git-3004", "stable-3005", "git-3005", "onedir-3005", "git-master", "latest"]' + instances: '["stable-3003", "git-3003", "stable-3004", "git-3004", "stable-3005", "git-3005", "onedir-3005", "git-master", "latest", "onedir-nightly"]' rockylinux-8: @@ -459,7 +459,7 @@ jobs: distro-slug: rockylinux-8 display-name: Rocky Linux 8 timeout: 20 - instances: '["stable-3004", "git-3004", "stable-3005", "git-3005", "onedir-3005", "git-master", "latest"]' + instances: '["stable-3004", "git-3004", "stable-3005", "git-3005", "onedir-3005", "git-master", "latest", "onedir-nightly"]' ubuntu-1804: @@ -473,7 +473,7 @@ jobs: distro-slug: ubuntu-1804 display-name: Ubuntu 18.04 timeout: 20 - instances: '["stable-3003", "git-3003", "stable-3004", "git-3004", "stable-3005", "git-3005", "onedir-3005", "git-master", "latest"]' + instances: '["stable-3003", "git-3003", "stable-3004", "git-3004", "stable-3005", "git-3005", "onedir-3005", "git-master", "latest", "onedir-nightly"]' ubuntu-2004: @@ -487,7 +487,7 @@ jobs: distro-slug: ubuntu-2004 display-name: Ubuntu 20.04 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "git-master", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly"]' ubuntu-2204: @@ -501,7 +501,7 @@ jobs: distro-slug: ubuntu-2204 display-name: Ubuntu 22.04 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "git-master", "latest"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly"]' set-pipeline-exit-status: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index a92dfbc..81ab8b7 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -66,6 +66,21 @@ STABLE_DISTROS = [ "ubuntu-2204", ] +ONEDIR_DISTROS = [ + "almalinux-8", + "amazon-2", + "centos-7", + "centos-stream8", + "debian-10", + "debian-11", + "oraclelinux-7", + "oraclelinux-8", + "rockylinux-8", + "ubuntu-1804", + "ubuntu-2004", + "ubuntu-2204", +] + BLACKLIST_3003 = [ "arch", "debian-11", @@ -154,11 +169,13 @@ SALT_VERSIONS = [ "3005", "master", "latest", + "nightly", ] ONEDIR_SALT_VERSIONS = [ "3005", "latest", + "nightly", ] VERSION_DISPLAY_NAMES = { @@ -167,10 +184,16 @@ VERSION_DISPLAY_NAMES = { "3005": "v3005", "master": "Master", "latest": "Latest", + "nightly": "Nightly", } STABLE_VERSION_BLACKLIST = [ "master", + "nightly", +] + +GIT_VERSION_BLACKLIST = [ + "nightly", ] LATEST_PKG_BLACKLIST = [] @@ -415,6 +438,8 @@ def generate_test_jobs(): if bootstrap_type == "onedir": if salt_version not in ONEDIR_SALT_VERSIONS: continue + if distro not in ONEDIR_DISTROS: + continue if bootstrap_type == "stable": if salt_version in STABLE_VERSION_BLACKLIST: @@ -426,6 +451,10 @@ def generate_test_jobs(): # Fedora does not keep old builds around continue + if bootstrap_type == "git": + if salt_version in GIT_VERSION_BLACKLIST: + continue + BLACKLIST = { "3003": BLACKLIST_3003, "3004": BLACKLIST_3004,