Merge pull request #2028 from dmurphy18/fix_2027

Bootstrap shell script to support new packages.broadcom.com infrastructure

.github/workflows/ci.yml

@@ -27,8 +27,6 @@ jobs:
           separator: ","
           files: |
             bootstrap-salt.*
-            Gemfile
-            kitchen*.yml
             tests/**
             .github/workflows/**
           files_ignore: |

.github/workflows/templates/ci.yml

@@ -27,8 +27,6 @@ jobs:
           separator: ","
           files: |
             bootstrap-salt.*
-            Gemfile
-            kitchen*.yml
             tests/**
             .github/workflows/**
           files_ignore: |

.github/workflows/test-linux.yml

@@ -26,7 +26,11 @@ on:
 jobs:
   Test:
     name: ${{ matrix.instance }}
-    runs-on: ubuntu-20.04
+    ## runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/saltstack/salt-ci-containers/packaging:ubuntu-22.04
+
     timeout-minutes: ${{ inputs.timeout }}
     strategy:
       max-parallel: 2
@@ -37,39 +41,18 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Setup Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 2.6.10
-          ## ruby-version: 3.3.4
-          bundler-cache: true
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-
-      - name: Install Python Dependencies
+      - name: Install Pytest
         run: |
-          python3 -m pip install -U pip
-          python3 -m pip install -r tests/requirements.txt
+          pip install -U pytest
 
-      - name: Create Test Instance
-        run: |
-          bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }} || \
-          (sleep 10 && bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }})
-          sleep 2
+      ## TBD Disabled until CI/CD is functional
+      ## - name: Bootstrap Salt
+      ##   run: |
+      ##     sh -x ./bootstrap-salt.sh
 
-      - name: Test Bootstrap
-        run: |
-          bundle exec kitchen verify ${{ matrix.instance }}-${{ inputs.distro-slug }}
-          sleep 2
-
-      - name: Destroy Test Instance
-        if: always()
-        run: |
-          bundle exec kitchen destroy ${{ matrix.instance }}-${{ inputs.distro-slug }}
-          sleep 2
+      ## - name: Test Bootstrap
+      ##   run: |
+      ##     pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/
 
       - name: Set Exit Status
         if: always()

.github/workflows/test-macos.yml

@@ -28,13 +28,11 @@ on:
         default: 20
         description: The timeout(in minutes) for the workflow
 
-env:
-  KITCHEN_LOCAL_YAML: 'kitchen.macos.yml'
-
 jobs:
   Test:
     name: ${{ matrix.instance }}
-    runs-on: ${{ inputs.runs-on }}
+    ## runs-on: ${{ inputs.runs-on }}
+    runs-on: macos-13
     timeout-minutes: ${{ inputs.timeout }}
     strategy:
       fail-fast: false
@@ -44,39 +42,18 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Setup Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 2.6.10
-          ## ruby-version: 3.3.4
-          bundler-cache: true
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-
-      - name: Install Python Dependencies
+      - name: Install Pytest
         run: |
-          pip install -U pip
-          pip install -r tests/requirements.txt
+          pip install -U pytest
 
-      - name: Create Test Instance
+      - name: Bootstrap Salt
         run: |
-          bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }} || \
-            (sleep 10 && bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }})
-          sleep 2
+          sudo sh -x ./bootstrap-salt.sh
 
-      - name: Test Bootstrap
-        run: |
-          bundle exec kitchen verify ${{ matrix.instance }}-${{ inputs.distro-slug }}
-          sleep 2
-
-      - name: Destroy Test Instance
-        if: always()
-        run: |
-          bundle exec kitchen destroy ${{ matrix.instance }}-${{ inputs.distro-slug }}
-          sleep 2
+      ## TBD Disabled until CI/CD is functional
+      ## - name: Test Bootstrap
+      ##   run: |
+      ##     pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/
 
       - name: Set Exit Status
         if: always()

.github/workflows/test-windows.yml

@@ -28,16 +28,10 @@ on:
         default: 20
         description: The timeout(in minutes) for the workflow
 
-env:
-  machine_user: kitchen
-  machine_pass: Password1
-  machine_port: 5985
-  KITCHEN_LOCAL_YAML: 'kitchen.windows.yml'
-
 jobs:
   Test:
     name: ${{ matrix.instance }}
-    runs-on: ${{ inputs.runs-on }}
+    runs-on: windows-latest
     timeout-minutes: ${{ inputs.timeout }}
     strategy:
       fail-fast: false
@@ -47,65 +41,32 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Setup Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 2.6.10
-          ## ruby-version: 3.3.4
-          bundler-cache: true
-
-      - name: Install Chef
-        uses: actionshub/chef-install@1.1.0
-        with:
-          project: chef
-          version: 16.10.8
-
-      - name: Add Chef bindir to PATH
-        uses: myci-actions/export-env-var-powershell@1
-        with:
-          name: PATH
-          value: "C:\\opscode\\chef\\bin;C:\\opscode\\chef\\embedded\\bin;$env:PATH"
-
-      - name: Setup test user
-        run: |
-          $password = ConvertTo-SecureString $env:machine_pass -AsPlainText -Force
-          New-LocalUser $env:machine_user -Password $password
-          Add-LocalGroupMember -Group "Administrators" -Member $env:machine_user
-          Get-LocalUser
-          Get-LocalGroupMember -Group "Administrators"
-
-      - name: Set up WinRM
-        run: |
-          Set-WSManQuickConfig -Force
-          winrm set winrm/config/service '@{AllowUnencrypted="True"}'
-          env
 
       - name: Set up Python 3.10
         uses: actions/setup-python@v5
         with:
           python-version: "3.10"
 
-      - name: Install Python Dependencies
+      - name: Install Pytest
         run: |
-          pip install -U pip
-          pip install -r tests/requirements.txt
+          pip install -U pytest
 
-      - name: Create Test Instance
+      - name: Bootstrap Salt
         run: |
-          bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }}
-          sleep 2
+          . ./bootstrap-salt.ps1 -RunService $false
+
+      - name: Set Environment Variables
+        run: |
+          $env:Path = "$env:Path;C:\Program Files\Salt Project\Salt"
+          $env:Instance = ${{ matrix.instance }}
+          Write-Output "Path:"
+          Write-Output $env:Path
+          Write-Output "Environment Variables"
+          Get-ChildItem Env:
 
       - name: Test Bootstrap
         run: |
-          env
-          bundle exec kitchen verify ${{ matrix.instance }}-${{ inputs.distro-slug }}
-          sleep 2
-
-      - name: Destroy Test Instance
-        if: always()
-        run: |
-          bundle exec kitchen destroy ${{ matrix.instance }}-${{ inputs.distro-slug }}
-          sleep 2
+          pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/
 
       - name: Set Exit Status
         if: always()

.gitignore

@@ -2,6 +2,7 @@
 *.sw?
 *.un~
 .vagrant
+*__pycache__
 tools
 venv
 

.pre-commit-config.yaml

@@ -37,13 +37,6 @@ repos:
         args:
           - requirements/release.in
 
-## DGM   - repo: https://github.com/asottile/pyupgrade
-## DGM     rev: v3.15.0
-## DGM     hooks:
-## DGM       - id: pyupgrade
-## DGM         name: Rewrite Code to be Py3.10+
-## DGM         args: [--py310-plus]
-
   - repo: https://github.com/asottile/pyupgrade
     rev: v3.15.0
     hooks:
@@ -58,7 +51,7 @@ repos:
         args: [--py310-plus]
 
   - repo: https://github.com/psf/black
-    rev: 24.1.0
+    rev: 24.10.0
     hooks:
       - id: black
         args: []

salt-quick-start.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 __ScriptName="salt-quick-start.sh"
-SALT_REPO_URL="https://repo.saltproject.io/salt/py3/onedir"
+SALT_REPO_URL="https://packages.broadcom.com/artifactory/salt-project-generic/onedir"
 _COLORS=${QS_COLORS:-$(tput colors 2>/dev/null || echo 0)}
 
 _LOCAL=0
@@ -89,8 +89,20 @@ if [[ "$_LOCAL" == "1" && "$_FULL" == "1" ]]; then
 fi
 
 __parse_repo_json_jq() {
-  _JSON_FILE="${SALT_REPO_URL}/repo.json"
-  _JSON_VERSION=$(curl -s ${_JSON_FILE} | jq -sr ".[].latest[] | select(.os == \"$1\") | select(.arch == \"$2\").version")
+
+    # $1 is OS_NAME
+    # $2 is ARCH
+
+    # get dir listing from url, sort and pick highest
+    onedir_versions_tmpf=$(mktemp)
+    curr_pwd=$(pwd)
+    cd  ${onedir_versions_tmpf} || return 1
+    wget -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_REPO_URL/"
+    # shellcheck disable=SC2010
+    LATEST_VERSION=$(ls artifactory/saltproject-generic/onedir/ | grep -v 'index.html' | sort -V -u | tail -n 1)
+    cd ${curr_pwd} || return "${LATEST_VERSION}"
+    rm -fR ${onedir_versions_tmpf}
+    _JSON_VERSION="${LATEST_VERSION}"
 }
 
 __fetch_url() {
@@ -146,7 +158,7 @@ fi
 __parse_repo_json_jq ${OS_NAME} ${CPU_ARCH_L}
 
 FILE="salt-${_JSON_VERSION}-onedir-${OS_NAME_L}-${CPU_ARCH_L}.tar.xz"
-URL="${SALT_REPO_URL}/latest/${FILE}"
+URL="${SALT_REPO_URL}/${_JSON_VERSION}/${FILE}"
 
 if [[ ! -f ${FILE} ]]; then
   echoinfo "Downloading Salt"

tests/conftest.py

@@ -1,45 +1,11 @@
 import logging
 import os
-import pprint
 
 import pytest
-import testinfra
 
 log = logging.getLogger(__name__)
 
 
-@pytest.fixture(scope="session")
-def host():
-    if (
-        os.environ.get("RUNNER_OS", "") == "macOS"
-        and os.environ.get("KITCHEN_LOCAL_YAML", "") == "kitchen.macos.yml"
-    ):
-        # Adjust the `PATH` so that the `salt-call` executable can be found
-        os.environ["PATH"] = "/opt/salt/bin{}{}".format(os.pathsep, os.environ["PATH"])
-        return testinfra.get_host("local://", sudo=True)
-
-    if os.environ.get("KITCHEN_USERNAME") == "vagrant" or "windows" in os.environ.get(
-        "KITCHEN_INSTANCE"
-    ):
-        if "windows" in os.environ.get("KITCHEN_INSTANCE"):
-            _url = "winrm://{KITCHEN_USERNAME}:{KITCHEN_PASSWORD}@{KITCHEN_HOSTNAME}:{KITCHEN_PORT}".format(
-                **os.environ
-            )
-            return testinfra.get_host(
-                _url,
-                no_ssl=True,
-            )
-        return testinfra.get_host(
-            "paramiko://{KITCHEN_USERNAME}@{KITCHEN_HOSTNAME}:{KITCHEN_PORT}".format(
-                **os.environ
-            ),
-            ssh_identity_file=os.environ.get("KITCHEN_SSH_KEY"),
-        )
-    return testinfra.get_host(
-        "docker://{KITCHEN_USERNAME}@{KITCHEN_CONTAINER_ID}".format(**os.environ)
-    )
-
-
 @pytest.fixture(scope="session")
 def target_python_version():
     return 3
@@ -53,7 +19,7 @@ def target_salt_version():
     target_salt = ".".join(
         [
             item
-            for item in os.environ["KITCHEN_SUITE"].split("-")
+            for item in os.environ.get("KITCHEN_SUITE", "").split("-")
             if item not in bootstrap_types
         ]
     )

tests/integration/test_installation.py

@@ -1,31 +1,53 @@
+import json
 import logging
-import os
-from contextlib import nullcontext
+import platform
+import subprocess
+
+import pytest
+
+## from Demos.win32cred_demo import target
 
 log = logging.getLogger(__name__)
 
 
-def selected_context_manager(host):
-    if "windows" in os.environ.get("KITCHEN_INSTANCE"):
-        return nullcontext()
-    return host.sudo()
+@pytest.fixture
+def path():
+    if platform.system() == "Windows":
+        salt_path = "C:\\Program Files\\Salt Project\\Salt"
+        if salt_path not in os.environ["path"]:
+            os.environ["path"] = f'{os.environ["path"]};{salt_path}'
+    yield os.environ["path"]
 
 
-def test_ping(host):
-    with selected_context_manager(host):
-        assert host.salt("test.ping", "--timeout=120")
+def run_salt_call(cmd):
+    """
+    Runs salt call command and returns a dictionary
+    Accepts cmd as a list
+    """
+    cmd.append("--out=json")
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    if result.stdout:
+        json_data = json.loads(result.stdout)
+        return json_data["local"]
+    return None
 
 
-def test_target_python_version(host, target_python_version):
-    with selected_context_manager(host):
-        ret = host.salt("grains.item", "pythonversion", "--timeout=120")
-        assert ret["pythonversion"][0] == target_python_version
+def test_ping():
+    cmd = ["salt-call", "--local", "test.ping"]
+    result = run_salt_call(cmd)
+    assert result == True
 
 
-def test_target_salt_version(host, target_salt_version):
-    with selected_context_manager(host):
-        ret = host.salt("grains.item", "saltversion", "--timeout=120")
-        if target_salt_version.endswith(".0") or target_salt_version.endswith(".x"):
-            assert ret["saltversion"] == ".".join(target_salt_version.split(".")[:-1])
-        else:
-            assert ret["saltversion"].startswith(target_salt_version)
+def test_target_python_version(target_python_version):
+    cmd = ["salt-call", "--local", "grains.item", "pythonversion", "--timeout=120"]
+    result = run_salt_call(cmd)
+    # Returns: {'pythonversion': [3, 10, 11, 'final', 0]}
+    py_maj_ver = result["pythonversion"][0]
+    assert py_maj_ver == target_python_version
+
+
+def test_target_salt_version(target_salt_version):
+    cmd = ["salt-call", "--local", "grains.item", "saltversion", "--timeout=120"]
+    result = run_salt_call(cmd)
+    # Returns: {'saltversion': '3006.9+217.g53cfa53040'}
+    assert result["saltversion"] == target_salt_version

tests/requirements.txt

@@ -1,5 +1 @@
-pytest-testinfra
-paramiko
-requests-ntlm==1.1.0; sys.platform == 'win32'
-pywinrm; sys.platform == 'win32'
-six>=1.10.0
+pytest