diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e88f30f..71d6220 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,6 +180,36 @@ jobs: + amazonlinux-2023: + name: Amazon 2023 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: amazonlinux-2023 + display-name: Amazon 2023 + container-slug: amazonlinux-2023 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + + debian-11: + name: Debian 11 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: debian-11 + display-name: Debian 11 + container-slug: debian-11 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + debian-12: name: Debian 12 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -190,7 +220,7 @@ jobs: with: distro-slug: debian-12 display-name: Debian 12 - container-slug: systemd-debian-12 + container-slug: debian-12 timeout: 20 instances: '["stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -205,7 +235,7 @@ jobs: with: distro-slug: photon-5 display-name: Photon OS 5 - container-slug: systemd-photon-5 + container-slug: photon-5 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -220,7 +250,7 @@ jobs: with: distro-slug: rockylinux-8 display-name: Rocky Linux 8 - container-slug: systemd-rockylinux-8 + container-slug: rockylinux-8 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -235,7 +265,7 @@ jobs: with: distro-slug: rockylinux-9 display-name: Rocky Linux 9 - container-slug: systemd-rockylinux-9 + container-slug: rockylinux-9 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -250,7 +280,7 @@ jobs: with: distro-slug: ubuntu-2204 display-name: Ubuntu 22.04 - container-slug: systemd-ubuntu-22.04 + container-slug: ubuntu-22.04 timeout: 20 instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -267,6 +297,8 @@ jobs: - macos-13 - macos-14 - windows-2022 + - amazonlinux-2023 + - debian-11 - debian-12 - photon-5 - rockylinux-8 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index ca31b4c..cb0e693 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -6,17 +6,15 @@ import pathlib os.chdir(os.path.abspath(os.path.dirname(__file__))) -# only test against current containers with systemd -# will add these when they become available with systemd # "amazonlinux-2", -# "amazonlinux-2023", -# "debian-11", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", LINUX_DISTROS = [ + "amazonlinux-2023", + "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -34,17 +32,15 @@ OSX = [ "macos-14", ] -# only test against current containers with systemd -# will add these when they become available with systemd # "amazonlinux-2", -# "amazonlinux-2023", -# "debian-11", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", STABLE_DISTROS = [ + "amazonlinux-2023", + "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -52,17 +48,15 @@ STABLE_DISTROS = [ "ubuntu-2204", ] -# only test against current containers with systemd -# will add these when they become available with systemd # "amazonlinux-2", -# "amazonlinux-2023", -# "debian-11", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", ONEDIR_DISTROS = [ + "amazonlinux-2023", + "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -70,8 +64,6 @@ ONEDIR_DISTROS = [ "ubuntu-2204", ] -# only test against current containers with systemd -# will add these when they become available with systemd # "amazonlinux-2", # "amazonlinux-2023", # "photon-4", @@ -102,6 +94,7 @@ BLACKLIST_GIT_3006 = [ "amazonlinux-2023", "debian-11", "debian-12", + "debian-13", "fedora-40", "photon-4", "photon-5", @@ -217,19 +210,19 @@ DISTRO_DISPLAY_NAMES = { } CONTAINER_SLUG_NAMES = { - "amazonlinux-2": "systemd-amazonlinux-2", - "amazonlinux-2023": "systemd-amazonlinux-2023", - "debian-11": "systemd-debian-11", - "debian-12": "systemd-debian-12", - "debian-13": "systemd-debian-13", - "fedora-40": "systemd-fedora-40", - "photon-4": "systemd-photon-4", - "photon-5": "systemd-photon-5", - "rockylinux-8": "systemd-rockylinux-8", - "rockylinux-9": "systemd-rockylinux-9", - "ubuntu-2004": "systemd-ubuntu-20.04", - "ubuntu-2204": "systemd-ubuntu-22.04", - "ubuntu-2404": "systemd-ubuntu-24.04", + "amazonlinux-2": "amazonlinux-2", + "amazonlinux-2023": "amazonlinux-2023", + "debian-11": "debian-11", + "debian-12": "debian-12", + "debian-13": "debian-13", + "fedora-40": "fedora-40", + "photon-4": "photon-4", + "photon-5": "photon-5", + "rockylinux-8": "rockylinux-8", + "rockylinux-9": "rockylinux-9", + "ubuntu-2004": "ubuntu-20.04", + "ubuntu-2204": "ubuntu-22.04", + "ubuntu-2404": "ubuntu-24.04", "macos-12": "macos-12", "macos-13": "macos-13", "macos-14": "macOS 14", diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 120f23e..b9c5662 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -32,10 +32,6 @@ jobs: Test: name: ${{ matrix.instance }} runs-on: ubuntu-latest - container: - image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} - options: --privileged - timeout-minutes: ${{ inputs.timeout }} strategy: max-parallel: 2 @@ -46,16 +42,6 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Python Dependencies with pip breakage - if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} - run: | - python3 -m pip install --break-system-packages -r tests/requirements.txt - - - name: Install Python Dependencies without pip breakage - if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} - run: | - python3 -m pip install -r tests/requirements.txt - - name: Get Version run: | # We need to get the version here and make it an environment variable @@ -65,17 +51,39 @@ jobs: vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + - name: "Pull container ${{ inputs.container-slug }}" + run: | + docker pull ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} + + - name: "Create container ${{ inputs.container-slug }}" + run: | + /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /_w/ --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -e $GITHUB_ENV -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" -v "/home/runner/work/salt-bootstrap/salt-bootstrap":"/_w/btstrap" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target + + - name: "Start container ${{ inputs.container-slug }}" + run: | + /usr/bin/docker start ${{ github.run_id }}_salt-test + + - name: "Install Python Dependencies with pip breakage in container ${{ inputs.container-slug }}" + if: ${{ ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} + run: | + docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r /_w/btstrap/tests/requirements.txt + + - name: "Install Python Dependencies without pip breakage in container ${{ inputs.container-slug }}" + if: ${{ ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} + run: | + docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r /_w/btstrap/tests/requirements.txt + - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot bt_arg1=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $1}') bt_arg2=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" + docker exec ${{ github.run_id}}_salt-test sh -x /_w/btstrap/bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap run: | - pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + docker exec ${{ github.run_id}}_salt-test pytest --cache-clear -v -s -ra --log-cli-level=debug /_w/btstrap/tests/integration/ - name: Set Exit Status if: always() diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 85e4d17..04a74c1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,8 +41,10 @@ repos: rev: v3.15.0 hooks: - id: pyupgrade - name: Rewrite Code to be Py3.9+ - args: [--py39-plus] + name: Rewrite Code to be Py3.6+ + args: [--py36-plus] + ## DGM name: Rewrite Code to be Py3.9+ + ## DGM args: [--py39-plus] - repo: https://github.com/asottile/reorder_python_imports rev: v3.12.0 diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 893205d..bd29313 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.12.09" +__ScriptVersion="2024.12.12" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -3189,11 +3189,8 @@ install_ubuntu_git_deps() { __PACKAGES="${__PACKAGES} util-linux-extra" fi - # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 - __PACKAGES="${__PACKAGES} procps pciutils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + # Additionally install procps pciutils and sudo which allows for Docker bootstraps. See 366#issuecomment-39666813 + __PACKAGES="${__PACKAGES} procps pciutils sudo" # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3592,15 +3589,12 @@ install_debian_onedir_deps() { return 1 fi - # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 - __PACKAGES='procps pciutils' + # Additionally install procps, pciutils and sudo which allows for Docker bootstraps. See 366#issuecomment-39666813 + __PACKAGES='procps pciutils sudo' # YAML module is used for generating custom master/minion configs __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-yaml" - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" - # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3643,11 +3637,8 @@ install_debian_git_deps() { __PACKAGES="python${PY_PKG_VER}-dev python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" echodebug "install_debian_git_deps() Installing ${__PACKAGES}" - # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 - __PACKAGES="${__PACKAGES} procps pciutils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + # Additionally install procps, pciutils and sudo which allows for Docker bootstraps. See 366#issuecomment-39666813 + __PACKAGES="${__PACKAGES} procps pciutils sudo" # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3947,14 +3938,11 @@ install_fedora_deps() { __PACKAGES="${__PACKAGES} dnf-utils libyaml procps-ng python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2" __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests python${PY_PKG_VER}-zmq" __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-pip python${PY_PKG_VER}-m2crypto python${PY_PKG_VER}-pyyaml" - __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd sudo" if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" fi - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" - # shellcheck disable=SC2086 __dnf_install_noinput ${__PACKAGES} ${_EXTRA_PACKAGES} || return 1 @@ -3985,10 +3973,7 @@ install_fedora_git_deps() { # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 - __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc gcc-c++" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc gcc-c++ sudo" # shellcheck disable=SC2086 __dnf_install_noinput ${__PACKAGES} || return 1 @@ -4117,10 +4102,7 @@ install_fedora_onedir_deps() { __install_saltstack_fedora_onedir_repository || return 1 fi - __PACKAGES="dnf-utils chkconfig procps-ng" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="dnf-utils chkconfig procps-ng sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4280,10 +4262,7 @@ install_centos_stable_deps() { __install_saltstack_rhel_onedir_repository || return 1 fi - __PACKAGES="yum-utils chkconfig procps-ng findutils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="yum-utils chkconfig procps-ng findutils sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4329,9 +4308,6 @@ install_centos_stable() { __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" - # shellcheck disable=SC2086 yum makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4410,10 +4386,7 @@ install_centos_git_deps() { return 1 fi - __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4509,10 +4482,7 @@ install_centos_onedir_deps() { __install_saltstack_rhel_onedir_repository || return 1 fi - __PACKAGES="yum-utils chkconfig procps-ng findutils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="yum-utils chkconfig procps-ng findutils sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5613,10 +5583,7 @@ install_amazon_linux_ami_2_git_deps() { # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 - __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5638,10 +5605,7 @@ install_amazon_linux_ami_2_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __PACKAGES="yum-utils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="yum-utils sudo" __yum_install_noinput ${__PACKAGES} @@ -5731,10 +5695,7 @@ install_amazon_linux_ami_2_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __PACKAGES="yum-utils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="yum-utils chkconfig procps-ng findutils sudo" __yum_install_noinput ${__PACKAGES} @@ -5891,10 +5852,7 @@ install_amazon_linux_ami_2023_git_deps() { # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 - __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5919,10 +5877,7 @@ install_amazon_linux_ami_2023_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __PACKAGES="yum-utils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="yum-utils chkconfig procps-ng findutils sudo" __yum_install_noinput ${__PACKAGES} diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 38e6e81..2bc7394 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -35,10 +35,21 @@ def run_salt_call(cmd): log.error(f"failed to produce output result, '{result}'") else: - cmdl = ["sudo"] + if platform.system() == "Darwin": + cmdl = ["sudo"] + else: + cmdl = [] cmdl.extend(cmd) cmdl.append("--out=json") - result = subprocess.run(cmdl, capture_output=True, text=True) + try: + result = subprocess.run(cmdl, capture_output=True, text=True) + except TypeError: + result = subprocess.run( + cmdl, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) if 0 == result.returncode: json_data = json.loads(result.stdout) else: