Pass along the relenv version that should be used.

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

.github/workflows/build-deb-packages.yml

@@ -8,6 +8,10 @@ on:
         type: string
         required: true
         description: The Salt version to set prior to building packages.
+      relenv-version:
+        type: string
+        required: true
+        description: The relenv version to set prior to building packages.
 
 jobs:
   build:
@@ -80,7 +84,7 @@ jobs:
       - name: Build Deb
         working-directory: pkgs/checkout/
         run: |
-          tools pkg build deb ${{
+          tools pkg build deb --relenv-version=${{ inputs.relenv-version }} ${{
               matrix.source == 'onedir' &&
               format('--onedir=salt-{0}-onedir-linux-{1}.tar.xz', inputs.salt-version, matrix.arch)
               ||

.github/workflows/build-macos-packages.yml

@@ -8,6 +8,10 @@ on:
         type: string
         required: true
         description: The Salt version to set prior to building packages.
+      relenv-version:
+        type: string
+        required: true
+        description: The relenv version to set prior to building packages.
       sign-packages:
         type: boolean
         default: false
@@ -104,7 +108,7 @@ jobs:
           APPLE_ACCT: "${{ secrets.MAC_SIGN_APPLE_ACCT }}"
           APP_SPEC_PWD: "${{ secrets.MAC_SIGN_APP_SPEC_PWD }}"
         run: |
-          tools pkg build macos ${{
+          tools pkg build macos --relenv-version=${{ inputs.relenv-version }}  ${{
               matrix.source == 'onedir' &&
               format(
                 '--onedir salt-{0}-onedir-darwin-{1}.tar.xz --salt-version {0} {2}',

.github/workflows/build-rpm-packages.yml

@@ -8,6 +8,10 @@ on:
         type: string
         required: true
         description: The Salt version to set prior to building packages.
+      relenv-version:
+        type: string
+        required: true
+        description: The relenv version to set prior to building packages.
 
 env:
   COLUMNS: 190
@@ -68,7 +72,7 @@ jobs:
 
       - name: Build RPM
         run: |
-          tools pkg build rpm ${{
+          tools pkg build rpm  --relenv-version=${{ inputs.relenv-version }} ${{
               matrix.source == 'onedir' &&
               format('--onedir=salt-{0}-onedir-linux-{1}.tar.xz', inputs.salt-version, matrix.arch)
               ||

.github/workflows/build-windows-packages.yml

@@ -8,6 +8,10 @@ on:
         type: string
         required: true
         description: The Salt version to set prior to building packages
+      relenv-version:
+        type: string
+        required: true
+        description: The relenv version to set prior to building packages.
       sign-packages:
         type: boolean
         default: false
@@ -99,7 +103,7 @@ jobs:
 
       - name: Build Windows Packages
         run: |
-          tools pkg build windows ${{
+          tools pkg build windows --relenv-version=${{ inputs.relenv-version }}  ${{
             matrix.source == 'onedir' &&
             format(
               '--onedir salt-{0}-onedir-windows-{1}.zip --salt-version {0} --arch {1} {2}',

.github/workflows/ci.yml

@@ -480,6 +480,7 @@ jobs:
     uses: ./.github/workflows/build-rpm-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "0.12.3"
 
   build-deb-pkgs:
     name: Build DEB Packages
@@ -490,6 +491,7 @@ jobs:
     uses: ./.github/workflows/build-deb-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "0.12.3"
 
   build-windows-pkgs:
     name: Build Windows Packages
@@ -500,6 +502,7 @@ jobs:
     uses: ./.github/workflows/build-windows-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "0.12.3"
 
   build-macos-pkgs:
     name: Build macOS Packages
@@ -510,6 +513,7 @@ jobs:
     uses: ./.github/workflows/build-macos-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "0.12.3"
 
   amazonlinux-2-pkg-tests:
     name: Amazon Linux 2 Package Tests

.github/workflows/nightly.yml

@@ -538,6 +538,7 @@ jobs:
     uses: ./.github/workflows/build-rpm-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "0.12.3"
 
   build-deb-pkgs:
     name: Build DEB Packages
@@ -548,6 +549,7 @@ jobs:
     uses: ./.github/workflows/build-deb-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "0.12.3"
 
   build-windows-pkgs:
     name: Build Windows Packages
@@ -558,6 +560,7 @@ jobs:
     uses: ./.github/workflows/build-windows-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "0.12.3"
       environment: nightly
       sign-packages: false
     secrets: inherit
@@ -571,6 +574,7 @@ jobs:
     uses: ./.github/workflows/build-macos-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "0.12.3"
       environment: nightly
       sign-packages: true
     secrets: inherit

.github/workflows/scheduled.yml

@@ -523,6 +523,7 @@ jobs:
     uses: ./.github/workflows/build-rpm-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "0.12.3"
 
   build-deb-pkgs:
     name: Build DEB Packages
@@ -533,6 +534,7 @@ jobs:
     uses: ./.github/workflows/build-deb-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "0.12.3"
 
   build-windows-pkgs:
     name: Build Windows Packages
@@ -543,6 +545,7 @@ jobs:
     uses: ./.github/workflows/build-windows-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "0.12.3"
 
   build-macos-pkgs:
     name: Build macOS Packages
@@ -553,6 +556,7 @@ jobs:
     uses: ./.github/workflows/build-macos-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "0.12.3"
 
   amazonlinux-2-pkg-tests:
     name: Amazon Linux 2 Package Tests

.github/workflows/staging.yml

@@ -523,6 +523,7 @@ jobs:
     uses: ./.github/workflows/build-rpm-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "0.12.3"
 
   build-deb-pkgs:
     name: Build DEB Packages
@@ -533,6 +534,7 @@ jobs:
     uses: ./.github/workflows/build-deb-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "0.12.3"
 
   build-windows-pkgs:
     name: Build Windows Packages
@@ -543,6 +545,7 @@ jobs:
     uses: ./.github/workflows/build-windows-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "0.12.3"
       environment: staging
       sign-packages: ${{ inputs.sign-windows-packages }}
     secrets: inherit
@@ -556,6 +559,7 @@ jobs:
     uses: ./.github/workflows/build-macos-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "0.12.3"
       environment: staging
       sign-packages: true
     secrets: inherit

.github/workflows/templates/build-packages.yml.jinja

@@ -16,6 +16,7 @@
     uses: ./.github/workflows/build-<{ pkg_type }>-packages.yml
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      relenv-version: "<{ relenv_version }>"
     <%- if pkg_type in ("macos", "windows") and gh_environment %>
       environment: <{ gh_environment }>
       <%- if pkg_type == "macos" %>

.pre-commit-config.yaml

@@ -88,6 +88,7 @@ repos:
           - pyyaml==6.0
           - jinja2==3.1.2
           - packaging==23.0
+          - virustotal3==1.0.8
 
   - repo: https://github.com/saltstack/pip-tools-compile-impersonate
     rev: "4.8"

pkg/debian/rules

@@ -18,6 +18,7 @@ override_dh_auto_build:
 	mkdir -p build/onedir
 	python3 -m venv --clear --copies build/onedir/venv
 	build/onedir/venv/bin/python3 -m pip install relenv==$${SALT_RELENV_VERSION}
+	export FETCH_RELENV_VERSION=$${SALT_RELENV_VERSION}
 	export PY=$$(build/onedir/venv/bin/python3 -c 'import sys; sys.stdout.write("{}.{}".format(*sys.version_info)); sys.stdout.flush()') \
 		&& build/onedir/venv/bin/python3 -m pip install -r requirements/static/ci/py$${PY}/tools.txt
 	build/onedir/venv/bin/relenv fetch

pkg/macos/build_python.sh

@@ -46,6 +46,7 @@ _usage() {
      echo "  -h, --help      this message"
      echo "  -b, --build     build python instead of fetching"
      echo "  -v, --version   version of python to install, must be available with relenv"
+     echo "  -r, --relenv-version   version of python to install, must be available with relenv"
      echo ""
      echo "  To build python 3.10.11:"
      echo "      example: $0 --version 3.10.11"
@@ -85,7 +86,12 @@ while true; do
             ;;
         -v | --version )
             shift
-            PY_VERSION="$*"
+            PY_VERSION="$1"
+            shift
+            ;;
+        -r | --relenv-version )
+            shift
+            RELENV_VERSION="$1"
             shift
             ;;
         -b | --build )
@@ -99,8 +105,9 @@ while true; do
             exit 1
             ;;
         * )
-            PY_VERSION="$*"
+            echo "Invalid Arguments: $*"
-            shift
+            _usage
+            exit 1
             ;;
     esac
 done
@@ -183,8 +190,13 @@ fi
 # Installing Relenv
 #-------------------------------------------------------------------------------
 _msg "Installing relenv"
-pip install relenv >/dev/null 2>&1
+if [ -n "${RELENV_VERSION}" ]; then
-if [ -n "$(pip show relenv)" ]; then
+    pip install relenv==${RELENV_VERSION} >/dev/null 2>&1
+    export RELENV_FETCH_VERSION=${RELENV_VERSION}
+else
+    pip install relenv >/dev/null 2>&1
+fi
+if [ -n "$(relenv --version)" ]; then
     _success
 else
     _failure

pkg/rpm/salt.spec

@@ -137,6 +137,7 @@ cd $RPM_BUILD_DIR
 %if "%{getenv:SALT_ONEDIR_ARCHIVE}" == ""
   python3 -m venv --clear --copies build/venv
   build/venv/bin/python3 -m pip install relenv==${SALT_RELENV_VERSION}
+  export FETCH_RELENV_VERSION=${SALT_RELENV_VERSION}
   export PY=$(build/venv/bin/python3 -c 'import sys; sys.stdout.write("{}.{}".format(*sys.version_info)); sys.stdout.flush()')
   build/venv/bin/python3 -m pip install -r %{_salt_src}/requirements/static/ci/py${PY}/tools.txt
   build/venv/bin/relenv fetch

pkg/windows/build.ps1

@@ -39,17 +39,16 @@ param(
     [Parameter(Mandatory=$false)]
     [ValidatePattern("^\d{1,2}.\d{1,2}.\d{1,2}$")]
     [ValidateSet(
-        "3.11.2",
+        "3.11.3",
         "3.10.11"
     )]
     [Alias("p")]
-    # The version of Python to be built. Pythonnet only supports up to Python
-    # 3.8 for now. Pycurl stopped building wheel files after 7.43.0.5 which
-    # supported up to 3.8. So we're pinned to the latest version of Python 3.8.
-    # We may have to drop support for pycurl.
-    # Default is: 3.8.16
     [String] $PythonVersion = "3.10.11",
 
+    [Parameter(Mandatory=$false)]
+    [Alias("r")]
+    [String] $RelenvVersion = "0.12.3",
+
     [Parameter(Mandatory=$false)]
     [Alias("b")]
     # Build python from source instead of fetching a tarball
@@ -111,6 +110,7 @@ Write-Host $("#" * 80)
 Write-Host "Build Salt Installer Packages" -ForegroundColor Cyan
 Write-Host "- Salt Version:   $Version"
 Write-Host "- Python Version: $PythonVersion"
+Write-Host "- Relenv Version: $RelenvVersion"
 Write-Host "- Architecture:   $Architecture"
 Write-Host $("v" * 80)
 
@@ -165,6 +165,7 @@ if ( ! $SkipInstall ) {
   $KeywordArguments = @{
       Version = $PythonVersion
       Architecture = $Architecture
+      RelenvVersion = $RelenvVersion
   }
   if ( $Build ) {
       $KeywordArguments["Build"] = $false

pkg/windows/build_python.ps1

@@ -18,17 +18,16 @@ param(
     [Parameter(Mandatory=$false)]
     [ValidatePattern("^\d{1,2}.\d{1,2}.\d{1,2}$")]
     [ValidateSet(
-        "3.11.2",
+        "3.11.3",
         "3.10.11"
     )]
     [Alias("v")]
-    # The version of Python to be built. Pythonnet only supports up to Python
-    # 3.8 for now. Pycurl stopped building wheel files after 7.43.0.5 which
-    # supported up to 3.8. So we're pinned to the latest version of Python 3.8.
-    # We may have to drop support for pycurl or build it ourselves.
-    # Default is: 3.8.16
     [String] $Version = "3.10.11",
 
+    [Parameter(Mandatory=$false)]
+    [Alias("r")]
+    [String] $RelenvVersion = "0.12.3",
+
     [Parameter(Mandatory=$false)]
     [ValidateSet("x64", "x86", "amd64")]
     [Alias("a")]
@@ -85,6 +84,7 @@ if ( $Build ) {
 }
 Write-Host "$SCRIPT_MSG" -ForegroundColor Cyan
 Write-Host "- Python Version: $Version"
+Write-Host "- Relenv Version: $RelenvVersion"
 Write-Host "- Architecture:   $Architecture"
 Write-Host "- Build:          $Build"
 Write-Host $("-" * 80)
@@ -227,7 +227,7 @@ if ( $env:VIRTUAL_ENV ) {
 # Installing Relenv
 #-------------------------------------------------------------------------------
 Write-Host "Installing Relenv: " -NoNewLine
-pip install relenv --disable-pip-version-check | Out-Null
+pip install relenv==$RelenvVersion --disable-pip-version-check | Out-Null
 $output = pip list --disable-pip-version-check
 if ("relenv" -in $output.split()) {
     Write-Result "Success" -ForegroundColor Green
@@ -235,6 +235,7 @@ if ("relenv" -in $output.split()) {
     Write-Result "Failed" -ForegroundColor Red
     exit 1
 }
+$env:RELENV_FETCH_VERSION=$RelenvVersion
 
 #-------------------------------------------------------------------------------
 # Building Python with Relenv

tools/pkg/build.py

@@ -80,11 +80,18 @@ def debian(
             ctx.exit(1)
         ctx.info("Building the package from the source files")
         shared_constants = _get_shared_constants()
+        if not python_version:
+            python_version = shared_constants["python_version_linux"]
+        if not relenv_version:
+            relenv_version = shared_constants["relenv_version"]
+        if TYPE_CHECKING:
+            assert python_version
+            assert relenv_version
         new_env = {
-            "SALT_RELENV_VERSION": relenv_version or shared_constants["relenv_version"],
+            "SALT_RELENV_VERSION": relenv_version,
-            "SALT_PYTHON_VERSION": python_version
+            "SALT_PYTHON_VERSION": python_version,
-            or shared_constants["python_version_linux"],
             "SALT_PACKAGE_ARCH": str(arch),
+            "RELENV_FETCH_VERSION": relenv_version,
         }
         for key, value in new_env.items():
             os.environ[key] = value
@@ -140,11 +147,18 @@ def rpm(
             ctx.exit(1)
         ctx.info(f"Building the package from the source files")
         shared_constants = _get_shared_constants()
+        if not python_version:
+            python_version = shared_constants["python_version_linux"]
+        if not relenv_version:
+            relenv_version = shared_constants["relenv_version"]
+        if TYPE_CHECKING:
+            assert python_version
+            assert relenv_version
         new_env = {
-            "SALT_RELENV_VERSION": relenv_version or shared_constants["relenv_version"],
+            "SALT_RELENV_VERSION": relenv_version,
-            "SALT_PYTHON_VERSION": python_version
+            "SALT_PYTHON_VERSION": python_version,
-            or shared_constants["python_version_linux"],
             "SALT_PACKAGE_ARCH": str(arch),
+            "RELENV_FETCH_VERSION": relenv_version,
         }
         for key, value in new_env.items():
             os.environ[key] = value
@@ -171,10 +185,21 @@ def rpm(
         "sign": {
             "help": "Sign and notorize built package",
         },
+        "relenv_version": {
+            "help": "The version of relenv to use",
+        },
+        "python_version": {
+            "help": "The version of python to build with using relenv",
+        },
     },
 )
 def macos(
-    ctx: Context, onedir: str = None, salt_version: str = None, sign: bool = False
+    ctx: Context,
+    onedir: str = None,
+    salt_version: str = None,
+    sign: bool = False,
+    relenv_version: str = None,
+    python_version: str = None,
 ):
     """
     Build the macOS package.
@@ -201,10 +226,23 @@ def macos(
     if not onedir:
         # Prep the salt onedir if not building from an existing one
         shared_constants = _get_shared_constants()
-        py_ver = shared_constants["python_version_macos"]
+        if not python_version:
+            python_version = shared_constants["python_version_linux"]
+        if not relenv_version:
+            relenv_version = shared_constants["relenv_version"]
+        if TYPE_CHECKING:
+            assert python_version
+            assert relenv_version
+        os.environ["RELENV_FETCH_VERSION"] = relenv_version
         with ctx.chdir(checkout / "pkg" / "macos"):
             ctx.info("Fetching relenv python")
-            ctx.run("./build_python.sh", "--version", py_ver)
+            ctx.run(
+                "./build_python.sh",
+                "--version",
+                python_version,
+                "--relenv-version",
+                relenv_version,
+            )
 
             ctx.info("Installing salt into the relenv python")
             ctx.run("./install_salt.sh")
@@ -252,6 +290,12 @@ def macos(
         "sign": {
             "help": "Sign and notarize built package",
         },
+        "relenv_version": {
+            "help": "The version of relenv to use",
+        },
+        "python_version": {
+            "help": "The version of python to build with using relenv",
+        },
     },
 )
 def windows(
@@ -260,6 +304,8 @@ def windows(
     salt_version: str = None,
     arch: str = None,
     sign: bool = False,
+    relenv_version: str = None,
+    python_version: str = None,
 ):
     """
     Build the Windows package.
@@ -268,6 +314,16 @@ def windows(
         assert salt_version is not None
         assert arch is not None
 
+    shared_constants = _get_shared_constants()
+    if not python_version:
+        python_version = shared_constants["python_version_linux"]
+    if not relenv_version:
+        relenv_version = shared_constants["relenv_version"]
+    if TYPE_CHECKING:
+        assert python_version
+        assert relenv_version
+    os.environ["RELENV_FETCH_VERSION"] = relenv_version
+
     build_cmd = [
         "powershell.exe",
         "&",
@@ -276,6 +332,10 @@ def windows(
         arch,
         "-Version",
         salt_version,
+        "-PythonVersion",
+        python_version,
+        "-RelenvVersion",
+        relenv_version,
         "-CICD",
     ]
 
@@ -383,6 +443,9 @@ def windows(
             "help": "The version of python to create an environment for using relenv",
             "required": True,
         },
+        "relenv_version": {
+            "help": "The version of relenv to use",
+        },
         "package_name": {
             "help": "The name of the relenv environment to be created",
             "required": True,
@@ -397,6 +460,7 @@ def onedir_dependencies(
     ctx: Context,
     arch: str = None,
     python_version: str = None,
+    relenv_version: str = None,
     package_name: str = None,
     platform: str = None,
 ):
@@ -411,6 +475,16 @@ def onedir_dependencies(
         assert package_name is not None
         assert platform is not None
 
+    shared_constants = _get_shared_constants()
+    if not python_version:
+        python_version = shared_constants[f"python_version_{platform}"]
+    if not relenv_version:
+        relenv_version = shared_constants["relenv_version"]
+    if TYPE_CHECKING:
+        assert python_version
+        assert relenv_version
+    os.environ["RELENV_FETCH_VERSION"] = relenv_version
+
     # We import relenv here because it is not a hard requirement for the rest of the tools commands
     try:
         import relenv.create
@@ -433,12 +507,11 @@ def onedir_dependencies(
         ctx.error(f"Failed to get the relenv version: {ret}")
         ctx.exit(1)
 
-    target_relenv_version = _get_shared_constants()["relenv_version"]
     env_relenv_version = ret.stdout.strip().decode()
-    if env_relenv_version != target_relenv_version:
+    if env_relenv_version != relenv_version:
         ctx.error(
             f"The onedir installed relenv version({env_relenv_version}) is not "
-            f"the relenv version which should be used({target_relenv_version})."
+            f"the relenv version which should be used({relenv_version})."
         )
         ctx.exit(1)
 
@@ -529,6 +602,9 @@ def onedir_dependencies(
             "help": "The name of the relenv environment to install salt into",
             "required": True,
         },
+        "relenv_version": {
+            "help": "The version of relenv to use",
+        },
     },
 )
 def salt_onedir(
@@ -536,6 +612,7 @@ def salt_onedir(
     salt_name: str,
     platform: str = None,
     package_name: str = None,
+    relenv_version: str = None,
 ):
     """
     Install salt into a relenv onedir environment.
@@ -544,6 +621,13 @@ def salt_onedir(
         assert platform is not None
         assert package_name is not None
 
+    shared_constants = _get_shared_constants()
+    if not relenv_version:
+        relenv_version = shared_constants["relenv_version"]
+    if TYPE_CHECKING:
+        assert relenv_version
+    os.environ["RELENV_FETCH_VERSION"] = relenv_version
+
     salt_archive = pathlib.Path(salt_name).resolve()
     onedir_env = pathlib.Path(package_name).resolve()
     _check_pkg_build_files_exist(ctx, onedir_env=onedir_env, salt_archive=salt_archive)
@@ -561,12 +645,11 @@ def salt_onedir(
         ctx.error(f"Failed to get the relenv version: {ret}")
         ctx.exit(1)
 
-    target_relenv_version = _get_shared_constants()["relenv_version"]
     env_relenv_version = ret.stdout.strip().decode()
-    if env_relenv_version != target_relenv_version:
+    if env_relenv_version != relenv_version:
         ctx.error(
             f"The onedir installed relenv version({env_relenv_version}) is not "
-            f"the relenv version which should be used({target_relenv_version})."
+            f"the relenv version which should be used({relenv_version})."
         )
         ctx.exit(1)