Merge pull request #2071 from dmurphy18/adj_containers_used

Updating container usage to that used in new Salt pipelines for CI/CD, fix Amazon Linux
This commit is contained in:
David Murphy 2024-12-12 13:15:24 -07:00 committed by GitHub
commit e85ff18bbc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 116 additions and 115 deletions

View file

@ -180,6 +180,36 @@ jobs:
amazonlinux-2023:
name: Amazon 2023
if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true'
uses: ./.github/workflows/test-linux.yml
needs:
- lint
- generate-actions-workflow
with:
distro-slug: amazonlinux-2023
display-name: Amazon 2023
container-slug: amazonlinux-2023
timeout: 20
instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]'
debian-11:
name: Debian 11
if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true'
uses: ./.github/workflows/test-linux.yml
needs:
- lint
- generate-actions-workflow
with:
distro-slug: debian-11
display-name: Debian 11
container-slug: debian-11
timeout: 20
instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]'
debian-12:
name: Debian 12
if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true'
@ -190,7 +220,7 @@ jobs:
with:
distro-slug: debian-12
display-name: Debian 12
container-slug: systemd-debian-12
container-slug: debian-12
timeout: 20
instances: '["stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]'
@ -205,7 +235,7 @@ jobs:
with:
distro-slug: photon-5
display-name: Photon OS 5
container-slug: systemd-photon-5
container-slug: photon-5
timeout: 20
instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]'
@ -220,7 +250,7 @@ jobs:
with:
distro-slug: rockylinux-8
display-name: Rocky Linux 8
container-slug: systemd-rockylinux-8
container-slug: rockylinux-8
timeout: 20
instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]'
@ -235,7 +265,7 @@ jobs:
with:
distro-slug: rockylinux-9
display-name: Rocky Linux 9
container-slug: systemd-rockylinux-9
container-slug: rockylinux-9
timeout: 20
instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]'
@ -250,7 +280,7 @@ jobs:
with:
distro-slug: ubuntu-2204
display-name: Ubuntu 22.04
container-slug: systemd-ubuntu-22.04
container-slug: ubuntu-22.04
timeout: 20
instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]'
@ -267,6 +297,8 @@ jobs:
- macos-13
- macos-14
- windows-2022
- amazonlinux-2023
- debian-11
- debian-12
- photon-5
- rockylinux-8

View file

@ -6,17 +6,15 @@ import pathlib
os.chdir(os.path.abspath(os.path.dirname(__file__)))
# only test against current containers with systemd
# will add these when they become available with systemd
# "amazonlinux-2",
# "amazonlinux-2023",
# "debian-11",
# "debian-13",
# "fedora-40",
# "photon-4",
# "ubuntu-2004",
# "ubuntu-2404",
LINUX_DISTROS = [
"amazonlinux-2023",
"debian-11",
"debian-12",
"photon-5",
"rockylinux-8",
@ -34,17 +32,15 @@ OSX = [
"macos-14",
]
# only test against current containers with systemd
# will add these when they become available with systemd
# "amazonlinux-2",
# "amazonlinux-2023",
# "debian-11",
# "debian-13",
# "fedora-40",
# "photon-4",
# "ubuntu-2004",
# "ubuntu-2404",
STABLE_DISTROS = [
"amazonlinux-2023",
"debian-11",
"debian-12",
"photon-5",
"rockylinux-8",
@ -52,17 +48,15 @@ STABLE_DISTROS = [
"ubuntu-2204",
]
# only test against current containers with systemd
# will add these when they become available with systemd
# "amazonlinux-2",
# "amazonlinux-2023",
# "debian-11",
# "debian-13",
# "fedora-40",
# "photon-4",
# "ubuntu-2004",
# "ubuntu-2404",
ONEDIR_DISTROS = [
"amazonlinux-2023",
"debian-11",
"debian-12",
"photon-5",
"rockylinux-8",
@ -70,8 +64,6 @@ ONEDIR_DISTROS = [
"ubuntu-2204",
]
# only test against current containers with systemd
# will add these when they become available with systemd
# "amazonlinux-2",
# "amazonlinux-2023",
# "photon-4",
@ -102,6 +94,7 @@ BLACKLIST_GIT_3006 = [
"amazonlinux-2023",
"debian-11",
"debian-12",
"debian-13",
"fedora-40",
"photon-4",
"photon-5",
@ -217,19 +210,19 @@ DISTRO_DISPLAY_NAMES = {
}
CONTAINER_SLUG_NAMES = {
"amazonlinux-2": "systemd-amazonlinux-2",
"amazonlinux-2023": "systemd-amazonlinux-2023",
"debian-11": "systemd-debian-11",
"debian-12": "systemd-debian-12",
"debian-13": "systemd-debian-13",
"fedora-40": "systemd-fedora-40",
"photon-4": "systemd-photon-4",
"photon-5": "systemd-photon-5",
"rockylinux-8": "systemd-rockylinux-8",
"rockylinux-9": "systemd-rockylinux-9",
"ubuntu-2004": "systemd-ubuntu-20.04",
"ubuntu-2204": "systemd-ubuntu-22.04",
"ubuntu-2404": "systemd-ubuntu-24.04",
"amazonlinux-2": "amazonlinux-2",
"amazonlinux-2023": "amazonlinux-2023",
"debian-11": "debian-11",
"debian-12": "debian-12",
"debian-13": "debian-13",
"fedora-40": "fedora-40",
"photon-4": "photon-4",
"photon-5": "photon-5",
"rockylinux-8": "rockylinux-8",
"rockylinux-9": "rockylinux-9",
"ubuntu-2004": "ubuntu-20.04",
"ubuntu-2204": "ubuntu-22.04",
"ubuntu-2404": "ubuntu-24.04",
"macos-12": "macos-12",
"macos-13": "macos-13",
"macos-14": "macOS 14",

View file

@ -32,10 +32,6 @@ jobs:
Test:
name: ${{ matrix.instance }}
runs-on: ubuntu-latest
container:
image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }}
options: --privileged
timeout-minutes: ${{ inputs.timeout }}
strategy:
max-parallel: 2
@ -46,16 +42,6 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Install Python Dependencies with pip breakage
if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }}
run: |
python3 -m pip install --break-system-packages -r tests/requirements.txt
- name: Install Python Dependencies without pip breakage
if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }}
run: |
python3 -m pip install -r tests/requirements.txt
- name: Get Version
run: |
# We need to get the version here and make it an environment variable
@ -65,17 +51,39 @@ jobs:
vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}')
echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV
- name: "Pull container ${{ inputs.container-slug }}"
run: |
docker pull ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }}
- name: "Create container ${{ inputs.container-slug }}"
run: |
/usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /_w/ --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -e $GITHUB_ENV -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" -v "/home/runner/work/salt-bootstrap/salt-bootstrap":"/_w/btstrap" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target
- name: "Start container ${{ inputs.container-slug }}"
run: |
/usr/bin/docker start ${{ github.run_id }}_salt-test
- name: "Install Python Dependencies with pip breakage in container ${{ inputs.container-slug }}"
if: ${{ ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }}
run: |
docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r /_w/btstrap/tests/requirements.txt
- name: "Install Python Dependencies without pip breakage in container ${{ inputs.container-slug }}"
if: ${{ ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }}
run: |
docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r /_w/btstrap/tests/requirements.txt
- name: Bootstrap Salt
run: |
# sed 1st - becomes space, 2nd - becomes dot
bt_arg1=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $1}')
bt_arg2=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}')
echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2,"
sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2"
docker exec ${{ github.run_id}}_salt-test sh -x /_w/btstrap/bootstrap-salt.sh "$bt_arg1" "$bt_arg2"
- name: Test Bootstrap
run: |
pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/
docker exec ${{ github.run_id}}_salt-test pytest --cache-clear -v -s -ra --log-cli-level=debug /_w/btstrap/tests/integration/
- name: Set Exit Status
if: always()

View file

@ -41,8 +41,10 @@ repos:
rev: v3.15.0
hooks:
- id: pyupgrade
name: Rewrite Code to be Py3.9+
args: [--py39-plus]
name: Rewrite Code to be Py3.6+
args: [--py36-plus]
## DGM name: Rewrite Code to be Py3.9+
## DGM args: [--py39-plus]
- repo: https://github.com/asottile/reorder_python_imports
rev: v3.12.0

View file

@ -26,7 +26,7 @@
#======================================================================================================================
set -o nounset # Treat unset variables as an error
__ScriptVersion="2024.12.09"
__ScriptVersion="2024.12.12"
__ScriptName="bootstrap-salt.sh"
__ScriptFullName="$0"
@ -3189,11 +3189,8 @@ install_ubuntu_git_deps() {
__PACKAGES="${__PACKAGES} util-linux-extra"
fi
# Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813
__PACKAGES="${__PACKAGES} procps pciutils"
# ensure sudo installed
__PACKAGES="${__PACKAGES} sudo"
# Additionally install procps pciutils and sudo which allows for Docker bootstraps. See 366#issuecomment-39666813
__PACKAGES="${__PACKAGES} procps pciutils sudo"
# shellcheck disable=SC2086
__apt_get_install_noinput ${__PACKAGES} || return 1
@ -3592,15 +3589,12 @@ install_debian_onedir_deps() {
return 1
fi
# Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813
__PACKAGES='procps pciutils'
# Additionally install procps, pciutils and sudo which allows for Docker bootstraps. See 366#issuecomment-39666813
__PACKAGES='procps pciutils sudo'
# YAML module is used for generating custom master/minion configs
__PACKAGES="${__PACKAGES} python${PY_PKG_VER}-yaml"
# ensure sudo installed
__PACKAGES="${__PACKAGES} sudo"
# shellcheck disable=SC2086
__apt_get_install_noinput ${__PACKAGES} || return 1
@ -3643,11 +3637,8 @@ install_debian_git_deps() {
__PACKAGES="python${PY_PKG_VER}-dev python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc"
echodebug "install_debian_git_deps() Installing ${__PACKAGES}"
# Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813
__PACKAGES="${__PACKAGES} procps pciutils"
# ensure sudo installed
__PACKAGES="${__PACKAGES} sudo"
# Additionally install procps, pciutils and sudo which allows for Docker bootstraps. See 366#issuecomment-39666813
__PACKAGES="${__PACKAGES} procps pciutils sudo"
# shellcheck disable=SC2086
__apt_get_install_noinput ${__PACKAGES} || return 1
@ -3947,14 +3938,11 @@ install_fedora_deps() {
__PACKAGES="${__PACKAGES} dnf-utils libyaml procps-ng python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2"
__PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests python${PY_PKG_VER}-zmq"
__PACKAGES="${__PACKAGES} python${PY_PKG_VER}-pip python${PY_PKG_VER}-m2crypto python${PY_PKG_VER}-pyyaml"
__PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd"
__PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd sudo"
if [ "${_EXTRA_PACKAGES}" != "" ]; then
echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}"
fi
# ensure sudo installed
__PACKAGES="${__PACKAGES} sudo"
# shellcheck disable=SC2086
__dnf_install_noinput ${__PACKAGES} ${_EXTRA_PACKAGES} || return 1
@ -3985,10 +3973,7 @@ install_fedora_git_deps() {
# shellcheck disable=SC2119
__git_clone_and_checkout || return 1
__PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc gcc-c++"
# ensure sudo installed
__PACKAGES="${__PACKAGES} sudo"
__PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc gcc-c++ sudo"
# shellcheck disable=SC2086
__dnf_install_noinput ${__PACKAGES} || return 1
@ -4117,10 +4102,7 @@ install_fedora_onedir_deps() {
__install_saltstack_fedora_onedir_repository || return 1
fi
__PACKAGES="dnf-utils chkconfig procps-ng"
# ensure sudo installed
__PACKAGES="${__PACKAGES} sudo"
__PACKAGES="dnf-utils chkconfig procps-ng sudo"
# shellcheck disable=SC2086
__yum_install_noinput ${__PACKAGES} || return 1
@ -4280,10 +4262,7 @@ install_centos_stable_deps() {
__install_saltstack_rhel_onedir_repository || return 1
fi
__PACKAGES="yum-utils chkconfig procps-ng findutils"
# ensure sudo installed
__PACKAGES="${__PACKAGES} sudo"
__PACKAGES="yum-utils chkconfig procps-ng findutils sudo"
# shellcheck disable=SC2086
__yum_install_noinput ${__PACKAGES} || return 1
@ -4329,9 +4308,6 @@ install_centos_stable() {
__PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG"
fi
# ensure sudo installed
__PACKAGES="${__PACKAGES} sudo"
# shellcheck disable=SC2086
yum makecache || return 1
__yum_install_noinput ${__PACKAGES} || return 1
@ -4410,10 +4386,7 @@ install_centos_git_deps() {
return 1
fi
__PACKAGES="${__PACKAGES} python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc"
# ensure sudo installed
__PACKAGES="${__PACKAGES} sudo"
__PACKAGES="${__PACKAGES} python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc sudo"
# shellcheck disable=SC2086
__yum_install_noinput ${__PACKAGES} || return 1
@ -4509,10 +4482,7 @@ install_centos_onedir_deps() {
__install_saltstack_rhel_onedir_repository || return 1
fi
__PACKAGES="yum-utils chkconfig procps-ng findutils"
# ensure sudo installed
__PACKAGES="${__PACKAGES} sudo"
__PACKAGES="yum-utils chkconfig procps-ng findutils sudo"
# shellcheck disable=SC2086
__yum_install_noinput ${__PACKAGES} || return 1
@ -5613,10 +5583,7 @@ install_amazon_linux_ami_2_git_deps() {
# shellcheck disable=SC2119
__git_clone_and_checkout || return 1
__PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc"
# ensure sudo installed
__PACKAGES="${__PACKAGES} sudo"
__PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc sudo"
# shellcheck disable=SC2086
__yum_install_noinput ${__PACKAGES} || return 1
@ -5638,10 +5605,7 @@ install_amazon_linux_ami_2_deps() {
# We need to install yum-utils before doing anything else when installing on
# Amazon Linux ECS-optimized images. See issue #974.
__PACKAGES="yum-utils"
# ensure sudo installed
__PACKAGES="${__PACKAGES} sudo"
__PACKAGES="yum-utils sudo"
__yum_install_noinput ${__PACKAGES}
@ -5731,10 +5695,7 @@ install_amazon_linux_ami_2_onedir_deps() {
# We need to install yum-utils before doing anything else when installing on
# Amazon Linux ECS-optimized images. See issue #974.
__PACKAGES="yum-utils"
# ensure sudo installed
__PACKAGES="${__PACKAGES} sudo"
__PACKAGES="yum-utils chkconfig procps-ng findutils sudo"
__yum_install_noinput ${__PACKAGES}
@ -5891,10 +5852,7 @@ install_amazon_linux_ami_2023_git_deps() {
# shellcheck disable=SC2119
__git_clone_and_checkout || return 1
__PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc"
# ensure sudo installed
__PACKAGES="${__PACKAGES} sudo"
__PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc sudo"
# shellcheck disable=SC2086
__yum_install_noinput ${__PACKAGES} || return 1
@ -5919,10 +5877,7 @@ install_amazon_linux_ami_2023_onedir_deps() {
# We need to install yum-utils before doing anything else when installing on
# Amazon Linux ECS-optimized images. See issue #974.
__PACKAGES="yum-utils"
# ensure sudo installed
__PACKAGES="${__PACKAGES} sudo"
__PACKAGES="yum-utils chkconfig procps-ng findutils sudo"
__yum_install_noinput ${__PACKAGES}

View file

@ -35,10 +35,21 @@ def run_salt_call(cmd):
log.error(f"failed to produce output result, '{result}'")
else:
cmdl = ["sudo"]
if platform.system() == "Darwin":
cmdl = ["sudo"]
else:
cmdl = []
cmdl.extend(cmd)
cmdl.append("--out=json")
result = subprocess.run(cmdl, capture_output=True, text=True)
try:
result = subprocess.run(cmdl, capture_output=True, text=True)
except TypeError:
result = subprocess.run(
cmdl,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
)
if 0 == result.returncode:
json_data = json.loads(result.stdout)
else: