From 2fa96841694ea88a4806c299166c37ed6fdf2ac2 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 28 Jun 2023 21:37:21 -0600 Subject: [PATCH 1/5] Fix an issue caching the installer when version=latest --- changelog/64519.fixed.md | 3 + salt/modules/win_pkg.py | 159 +++++++++----------- tests/pytests/unit/modules/test_win_pkg.py | 164 ++++++++++++++++----- 3 files changed, 194 insertions(+), 132 deletions(-) create mode 100644 changelog/64519.fixed.md diff --git a/changelog/64519.fixed.md b/changelog/64519.fixed.md new file mode 100644 index 00000000000..062109f6227 --- /dev/null +++ b/changelog/64519.fixed.md @@ -0,0 +1,3 @@ +`win_pkg` Fixes an issue runing `pkg.install` with `version=latest` where the +new installer would not be cached if there was already an installer present +with the same name. diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index e80dd193221..e8fdf22e419 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -215,7 +215,7 @@ def upgrade_available(name, **kwargs): refresh = salt.utils.data.is_true(kwargs.get("refresh", True)) # if latest_version returns blank, the latest version is already installed or - # their is no package definition. This is a salt standard which could be improved. + # there is no package definition. This is a salt standard which could be improved. return latest_version(name, saltenv=saltenv, refresh=refresh) != "" @@ -1315,7 +1315,7 @@ def _get_source_sum(source_hash, file_path, saltenv, verify_ssl=True): # The source_hash is a file on a server try: cached_hash_file = __salt__["cp.cache_file"]( - source_hash, saltenv=saltenv, verify_ssl=verify_ssl + source_hash, saltenv=saltenv, verify_ssl=verify_ssl, use_etag=True ) except MinionError as exc: log.exception("Failed to cache %s", source_hash, exc_info=exc) @@ -1640,6 +1640,13 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): ret[pkg_name] = {"no installer": version_num} continue + # Hash the installer source after verifying it was defined + installer_hash = __salt__["cp.hash_file"](installer, saltenv) + if isinstance(installer_hash, dict): + installer_hash = installer_hash["hsum"] + else: + installer_hash = None + # Is the installer in a location that requires caching if __salt__["config.valid_fileproto"](installer): @@ -1649,6 +1656,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # single files if cache_dir and installer.startswith("salt:"): path, _ = os.path.split(installer) + log.debug(f"PKG: Caching directory: {path}") try: __salt__["cp.cache_dir"]( path=path, @@ -1664,51 +1672,45 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # Check to see if the cache_file is cached... if passed if cache_file and cache_file.startswith("salt:"): + cache_file_hash = __salt__["cp.hash_file"](cache_file, saltenv) + log.debug(f"PKG: Caching file: {cache_file}") + try: + cached_file = __salt__["cp.cache_file"]( + cache_file, + saltenv=saltenv, + source_hash=cache_file_hash, + verify_ssl=kwargs.get("verify_ssl", True), + ) + except MinionError as exc: + msg = "Failed to cache {}".format(cache_file) + log.exception(msg, exc_info=exc) + return "{}\n{}".format(msg, exc) - # Check to see if the file is cached - cached_file = __salt__["cp.is_cached"](cache_file, saltenv) + # Check if the cache_file was cached successfully if not cached_file: - try: - cached_file = __salt__["cp.cache_file"]( - cache_file, - saltenv=saltenv, - verify_ssl=kwargs.get("verify_ssl", True), - ) - except MinionError as exc: - msg = "Failed to cache {}".format(cache_file) - log.exception(msg, exc_info=exc) - return "{}\n{}".format(msg, exc) + log.error("Unable to cache %s", cache_file) + ret[pkg_name] = {"failed to cache cache_file": cache_file} + continue - # Make sure the cached file is the same as the source - if __salt__["cp.hash_file"](cache_file, saltenv) != __salt__[ - "cp.hash_file" - ](cached_file): - try: - cached_file = __salt__["cp.cache_file"]( - cache_file, - saltenv=saltenv, - verify_ssl=kwargs.get("verify_ssl", True), - ) - except MinionError as exc: - msg = "Failed to cache {}".format(cache_file) - log.exception(msg, exc_info=exc) - return "{}\n{}".format(msg, exc) + # If version is "latest" we always cache because "cp.is_cached" only + # checks that the file exists, not that is has changed + cached_pkg = False + if version_num != "latest" and not installer.startswith("salt:"): + cached_pkg = __salt__["cp.is_cached"](installer, saltenv) - # Check if the cache_file was cached successfully - if not cached_file: - log.error("Unable to cache %s", cache_file) - ret[pkg_name] = {"failed to cache cache_file": cache_file} - continue - - # Check to see if the installer is cached - cached_pkg = __salt__["cp.is_cached"](installer, saltenv) if not cached_pkg: - # It's not cached. Cache it, mate. + # Since we're passing "installer_hash", it should only cache the + # file if the source_hash doesn't match, which only works on + # files hosted on "salt://". If the http/https url supports + # etag, it should also verify that information before caching + log.debug(f"PKG: Caching file: {installer}") try: cached_pkg = __salt__["cp.cache_file"]( installer, saltenv=saltenv, + source_hash=installer_hash, verify_ssl=kwargs.get("verify_ssl", True), + use_etag=True, ) except MinionError as exc: msg = "Failed to cache {}".format(installer) @@ -1722,29 +1724,6 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): ) ret[pkg_name] = {"unable to cache": installer} continue - - # Compare the hash of the cached installer to the source only if the - # file is hosted on salt: - if installer.startswith("salt:"): - if __salt__["cp.hash_file"](installer, saltenv) != __salt__[ - "cp.hash_file" - ](cached_pkg): - try: - cached_pkg = __salt__["cp.cache_file"]( - installer, - saltenv=saltenv, - verify_ssl=kwargs.get("verify_ssl", True), - ) - except MinionError as exc: - msg = "Failed to cache {}".format(installer) - log.exception(msg, exc_info=exc) - return "{}\n{}".format(msg, exc) - - # Check if the installer was cached successfully - if not cached_pkg: - log.error("Unable to cache %s", installer) - ret[pkg_name] = {"unable to cache": installer} - continue else: # Run the installer directly (not hosted on salt:, https:, etc.) cached_pkg = installer @@ -1786,7 +1765,6 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): log.debug("pkg.install: Source hash matches package hash.") # Get install flags - install_flags = pkginfo[version_num].get("install_flags", "") if options and options.get("extra_install_flags"): install_flags = "{} {}".format( @@ -2063,7 +2041,7 @@ def remove(name=None, pkgs=None, **kwargs): removal_targets.append(ver_install) else: if version_num in pkginfo: - # we known how to remove this version + # we know how to remove this version if version_num in old[pkgname]: removal_targets.append(version_num) else: @@ -2107,8 +2085,15 @@ def remove(name=None, pkgs=None, **kwargs): ret[pkgname] = {"no uninstaller defined": target} continue - # Where is the uninstaller - if uninstaller.startswith(("salt:", "http:", "https:", "ftp:")): + # Hash the uninstaller source after verifying it was defined + uninstaller_hash = __salt__["cp.hash_file"](uninstaller, saltenv) + if isinstance(uninstaller_hash, dict): + uninstaller_hash = uninstaller_hash["hsum"] + else: + uninstaller_hash = None + + # Is the uninstaller in a location that requires caching + if __salt__["config.valid_fileproto"](uninstaller): # Check for the 'cache_dir' parameter in the .sls file # If true, the entire directory will be cached instead of the @@ -2117,24 +2102,38 @@ def remove(name=None, pkgs=None, **kwargs): if cache_dir and uninstaller.startswith("salt:"): path, _ = os.path.split(uninstaller) + log.debug(f"PKG: Caching dir: {path}") try: __salt__["cp.cache_dir"]( - path, saltenv, False, None, "E@init.sls$" + path=path, + saltenv=saltenv, + include_empty=False, + include_pat=None, + exclude_pat="E@init.sls$", ) except MinionError as exc: msg = "Failed to cache {}".format(path) log.exception(msg, exc_info=exc) return "{}\n{}".format(msg, exc) - # Check to see if the uninstaller is cached + # Check to see if the uninstaller is cached. We don't want to + # check for latest here like we do for "pkg.install" because we + # only want to uninstall the version that has been installed cached_pkg = __salt__["cp.is_cached"](uninstaller, saltenv) if not cached_pkg: - # It's not cached. Cache it, mate. + # Since we're passing "uninstaller_hash", it should only + # cache the file if the source_hash doesn't match, which + # only works on files hosted on "salt://". If the http/https + # url supports etag, it should also verify that information + # before caching + log.debug(f"PKG: Caching file: {uninstaller}") try: cached_pkg = __salt__["cp.cache_file"]( uninstaller, saltenv=saltenv, + source_hash=uninstaller_hash, verify_ssl=kwargs.get("verify_ssl", True), + use_etag=True, ) except MinionError as exc: msg = "Failed to cache {}".format(uninstaller) @@ -2147,32 +2146,8 @@ def remove(name=None, pkgs=None, **kwargs): ret[pkgname] = {"unable to cache": uninstaller} continue - # Compare the hash of the cached installer to the source only if - # the file is hosted on salt: - # TODO cp.cache_file does cache and hash checking? So why do it again? - if uninstaller.startswith("salt:"): - if __salt__["cp.hash_file"](uninstaller, saltenv) != __salt__[ - "cp.hash_file" - ](cached_pkg): - try: - cached_pkg = __salt__["cp.cache_file"]( - uninstaller, - saltenv=saltenv, - verify_ssl=kwargs.get("verify_ssl", True), - ) - except MinionError as exc: - msg = "Failed to cache {}".format(uninstaller) - log.exception(msg, exc_info=exc) - return "{}\n{}".format(msg, exc) - - # Check if the installer was cached successfully - if not cached_pkg: - log.error("Unable to cache %s", uninstaller) - ret[pkgname] = {"unable to cache": uninstaller} - continue else: - # Run the uninstaller directly - # (not hosted on salt:, https:, etc.) + # Run the uninstaller directly (not hosted on salt:, https:, etc.) cached_pkg = os.path.expandvars(uninstaller) # Fix non-windows slashes diff --git a/tests/pytests/unit/modules/test_win_pkg.py b/tests/pytests/unit/modules/test_win_pkg.py index 6d435f00a54..9ef693a21f7 100644 --- a/tests/pytests/unit/modules/test_win_pkg.py +++ b/tests/pytests/unit/modules/test_win_pkg.py @@ -6,6 +6,7 @@ import logging import pytest import salt.modules.config as config +import salt.modules.cp as cp import salt.modules.pkg_resource as pkg_resource import salt.modules.win_pkg as win_pkg import salt.utils.data @@ -21,8 +22,17 @@ pytestmark = [ @pytest.fixture -def configure_loader_modules(): +def configure_loader_modules(minion_opts): pkg_info = { + "latest": { + "full_name": "Nullsoft Install System", + "installer": "http://download.sourceforge.net/project/nsis/nsis-setup.exe", + "install_flags": "/S", + "uninstaller": "%PROGRAMFILES(x86)%\\NSIS\\uninst-nsis.exe", + "uninstall_flags": "/S", + "msiexec": False, + "reboot": False, + }, "3.03": { "full_name": "Nullsoft Install System", "installer": "http://download.sourceforge.net/project/nsis/NSIS%203/3.03/nsis-3.03-setup.exe", @@ -43,16 +53,20 @@ def configure_loader_modules(): }, } + opts = minion_opts + opts["master_uri"] = "localhost" return { + cp: {"__opts__": opts}, win_pkg: { "_get_latest_package_version": MagicMock(return_value="3.03"), "_get_package_info": MagicMock(return_value=pkg_info), "__salt__": { + "config.valid_fileproto": config.valid_fileproto, + "cp.hash_file": cp.hash_file, "pkg_resource.add_pkg": pkg_resource.add_pkg, "pkg_resource.parse_targets": pkg_resource.parse_targets, "pkg_resource.sort_pkglist": pkg_resource.sort_pkglist, "pkg_resource.stringify": pkg_resource.stringify, - "config.valid_fileproto": config.valid_fileproto, }, "__utils__": { "reg.key_exists": win_reg.key_exists, @@ -164,19 +178,81 @@ def test_pkg_install_existing(): se_list_pkgs = {"nsis": ["3.03"]} with patch.object(win_pkg, "list_pkgs", return_value=se_list_pkgs), patch.object( win_pkg, "_get_reg_software", return_value=ret_reg - ), patch.dict( - win_pkg.__salt__, {"cp.is_cached": MagicMock(return_value=False)} ), patch.dict( win_pkg.__salt__, - {"cp.cache_file": MagicMock(return_value="C:\\fake\\path.exe")}, - ), patch.dict( - win_pkg.__salt__, {"cmd.run_all": MagicMock(return_value={"retcode": 0})} + { + "cmd.run_all": MagicMock(return_value={"retcode": 0}), + "cp.cache_file": MagicMock(return_value="C:\\fake\\path.exe"), + "cp.is_cached": MagicMock(return_value=True), + }, ): expected = {} result = win_pkg.install(name="nsis") assert expected == result +def test_pkg_install_latest(): + """ + test pkg.install when the package is already installed + no version passed + """ + ret_reg = {"Nullsoft Install System": "3.03"} + # The 2nd time it's run, pkg.list_pkgs uses with stringify + se_list_pkgs = [{"nsis": ["3.03"]}, {"nsis": "3.04"}] + mock_cache_file = MagicMock(return_value="C:\\fake\\path.exe") + with patch.object(win_pkg, "list_pkgs", side_effect=se_list_pkgs), patch.object( + win_pkg, "_get_reg_software", return_value=ret_reg + ), patch.dict( + win_pkg.__salt__, + { + "cmd.run_all": MagicMock(return_value={"retcode": 0}), + "cp.cache_file": mock_cache_file, + "cp.is_cached": MagicMock(return_value=False), + }, + ): + expected = {"nsis": {"new": "3.04", "old": "3.03"}} + result = win_pkg.install(name="nsis", version="latest") + assert expected == result + mock_cache_file.assert_called_once_with( + "http://download.sourceforge.net/project/nsis/nsis-setup.exe", + saltenv="base", + source_hash=None, + verify_ssl=True, + use_etag=True, + ) + + +def test_pkg_install_latest_is_cached(): + """ + test pkg.install when the package is already installed + no version passed + """ + ret_reg = {"Nullsoft Install System": "3.03"} + # The 2nd time it's run, pkg.list_pkgs uses with stringify + se_list_pkgs = [{"nsis": ["3.03"]}, {"nsis": "3.04"}] + mock_cache_file = MagicMock(return_value="C:\\fake\\path.exe") + with patch.object(win_pkg, "list_pkgs", side_effect=se_list_pkgs), patch.object( + win_pkg, "_get_reg_software", return_value=ret_reg + ), patch.dict( + win_pkg.__salt__, + { + "cmd.run_all": MagicMock(return_value={"retcode": 0}), + "cp.cache_file": mock_cache_file, + "cp.is_cached": MagicMock(return_value=True), + }, + ): + expected = {"nsis": {"new": "3.04", "old": "3.03"}} + result = win_pkg.install(name="nsis", version="latest") + assert expected == result + mock_cache_file.assert_called_once_with( + "http://download.sourceforge.net/project/nsis/nsis-setup.exe", + saltenv="base", + source_hash=None, + verify_ssl=True, + use_etag=True, + ) + + def test_pkg_install_existing_with_version(): """ test pkg.install when the package is already installed @@ -187,13 +263,13 @@ def test_pkg_install_existing_with_version(): se_list_pkgs = {"nsis": ["3.03"]} with patch.object(win_pkg, "list_pkgs", return_value=se_list_pkgs), patch.object( win_pkg, "_get_reg_software", return_value=ret_reg - ), patch.dict( - win_pkg.__salt__, {"cp.is_cached": MagicMock(return_value=False)} ), patch.dict( win_pkg.__salt__, - {"cp.cache_file": MagicMock(return_value="C:\\fake\\path.exe")}, - ), patch.dict( - win_pkg.__salt__, {"cmd.run_all": MagicMock(return_value={"retcode": 0})} + { + "cmd.run_all": MagicMock(return_value={"retcode": 0}), + "cp.cache_file": MagicMock(return_value="C:\\fake\\path.exe"), + "cp.is_cached": MagicMock(return_value=False), + }, ): expected = {} result = win_pkg.install(name="nsis", version="3.03") @@ -233,7 +309,7 @@ def test_pkg_install_name(): "cmd.run_all": mock_cmd_run_all, }, ): - ret = win_pkg.install( + win_pkg.install( name="firebox", version="3.03", extra_install_flags="-e True -test_flag True", @@ -248,22 +324,26 @@ def test_pkg_install_verify_ssl_false(): ret_reg = {"Nullsoft Install System": "3.03"} # The 2nd time it's run, pkg.list_pkgs uses with stringify se_list_pkgs = [{"nsis": ["3.03"]}, {"nsis": "3.02"}] - mock_cp = MagicMock(return_value="C:\\fake\\path.exe") + mock_cache_file = MagicMock(return_value="C:\\fake\\path.exe") with patch.object(win_pkg, "list_pkgs", side_effect=se_list_pkgs), patch.object( win_pkg, "_get_reg_software", return_value=ret_reg ), patch.dict( - win_pkg.__salt__, {"cp.is_cached": MagicMock(return_value=False)} - ), patch.dict( - win_pkg.__salt__, {"cp.cache_file": mock_cp} - ), patch.dict( - win_pkg.__salt__, {"cmd.run_all": MagicMock(return_value={"retcode": 0})} + win_pkg.__salt__, + { + "cmd.run_all": MagicMock(return_value={"retcode": 0}), + "cp.cache_file": mock_cache_file, + "cp.is_cached": MagicMock(return_value=False), + "cp.hash_file": MagicMock(return_value={"hsum": "abc123"}), + }, ): expected = {"nsis": {"new": "3.02", "old": "3.03"}} result = win_pkg.install(name="nsis", version="3.02", verify_ssl=False) - mock_cp.assert_called_once_with( + mock_cache_file.assert_called_once_with( "http://download.sourceforge.net/project/nsis/NSIS%203/3.02/nsis-3.02-setup.exe", saltenv="base", + source_hash="abc123", verify_ssl=False, + use_etag=True, ) assert expected == result @@ -300,7 +380,7 @@ def test_pkg_install_single_pkg(): "cmd.run_all": mock_cmd_run_all, }, ): - ret = win_pkg.install( + win_pkg.install( pkgs=["firebox"], version="3.03", extra_install_flags="-e True -test_flag True", @@ -387,7 +467,7 @@ def test_pkg_install_multiple_pkgs(): "cmd.run_all": mock_cmd_run_all, }, ): - ret = win_pkg.install( + win_pkg.install( pkgs=["firebox", "got"], extra_install_flags="-e True -test_flag True" ) assert "-e True -test_flag True" not in str(mock_cmd_run_all.call_args[0]) @@ -429,7 +509,7 @@ def test_pkg_install_minion_error_https(): "cp.cache_file": mock_minion_error, }, ): - ret = win_pkg.install( + result = win_pkg.install( name="firebox", version="3.03", ) @@ -438,7 +518,7 @@ def test_pkg_install_minion_error_https(): " getaddrinfo failed reading https://repo.test.com/runme.exe" ) - assert ret == expected + assert result == expected def test_pkg_install_minion_error_salt(): @@ -469,12 +549,13 @@ def test_pkg_install_minion_error_salt(): ), patch.dict( win_pkg.__salt__, { - "pkg_resource.parse_targets": mock_parse, - "cp.is_cached": mock_none, "cp.cache_file": mock_minion_error, + "cp.is_cached": mock_none, + "cp.hash_file": MagicMock(return_value={"hsum": "abc123"}), + "pkg_resource.parse_targets": mock_parse, }, ): - ret = win_pkg.install( + result = win_pkg.install( name="firebox", version="3.03", ) @@ -483,7 +564,7 @@ def test_pkg_install_minion_error_salt(): "Error: [Errno 1] failed reading salt://software/runme.exe" ) - assert ret == expected + assert result == expected def test_pkg_install_minion_error_salt_cache_dir(): @@ -505,18 +586,19 @@ def test_pkg_install_minion_error_salt_cache_dir(): } err_msg = "Error: [Errno 1] failed reading salt://software" - mock_none = MagicMock(return_value=None) mock_minion_error = MagicMock(side_effect=MinionError(err_msg)) - mock_parse = MagicMock(return_value=[{"firebox": "3.03"}, None]) with patch.object( salt.utils.data, "is_true", MagicMock(return_value=True) ), patch.object( win_pkg, "_get_package_info", MagicMock(return_value=ret__get_package_info) ), patch.dict( win_pkg.__salt__, - {"cp.cache_dir": mock_minion_error}, + { + "cp.cache_dir": mock_minion_error, + "cp.hash_file": MagicMock(return_value={"hsum": "abc123"}), + }, ): - ret = win_pkg.install( + result = win_pkg.install( name="firebox", version="3.03", ) @@ -525,7 +607,7 @@ def test_pkg_install_minion_error_salt_cache_dir(): "Error: [Errno 1] failed reading salt://software" ) - assert ret == expected + assert result == expected def test_pkg_remove_log_message(caplog): @@ -602,17 +684,18 @@ def test_pkg_remove_minion_error_salt_cache_dir(): ), patch.dict( win_pkg.__salt__, { - "pkg_resource.parse_targets": mock_parse, "cp.cache_dir": mock_minion_error, + "cp.hash_file": MagicMock(return_value={"hsum": "abc123"}), + "pkg_resource.parse_targets": mock_parse, }, ): - ret = win_pkg.remove(name="firebox") + result = win_pkg.remove(name="firebox") expected = ( "Failed to cache salt://software\n" "Error: [Errno 1] failed reading salt://software" ) - assert ret == expected + assert result == expected def test_pkg_remove_minion_error_salt(): @@ -644,18 +727,19 @@ def test_pkg_remove_minion_error_salt(): ), patch.dict( win_pkg.__salt__, { - "pkg_resource.parse_targets": mock_parse, - "cp.is_cached": mock_none, "cp.cache_file": mock_minion_error, + "cp.hash_file": MagicMock(return_value={"hsum": "abc123"}), + "cp.is_cached": mock_none, + "pkg_resource.parse_targets": mock_parse, }, ): - ret = win_pkg.remove(name="firebox") + result = win_pkg.remove(name="firebox") expected = ( "Failed to cache salt://software/runme.exe\n" "Error: [Errno 1] failed reading salt://software/runme.exe" ) - assert ret == expected + assert result == expected @pytest.mark.parametrize( From e235b82912143ae628e69b65deccae84edfe8b2e Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Thu, 29 Jun 2023 21:41:11 +0000 Subject: [PATCH 2/5] Update the bootstrap script to v2023.06.28 (cherry picked from commit 1ef02f5e1ddca5e8c80a3b7e378d035c125ba836) --- salt/cloud/deploy/bootstrap-salt.sh | 124 ++++++++++++++++++++++------ 1 file changed, 97 insertions(+), 27 deletions(-) diff --git a/salt/cloud/deploy/bootstrap-salt.sh b/salt/cloud/deploy/bootstrap-salt.sh index 6d69bf69213..1cf46c774df 100644 --- a/salt/cloud/deploy/bootstrap-salt.sh +++ b/salt/cloud/deploy/bootstrap-salt.sh @@ -23,7 +23,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2023.04.26" +__ScriptVersion="2023.06.28" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -224,7 +224,6 @@ _KEEP_TEMP_FILES=${BS_KEEP_TEMP_FILES:-$BS_FALSE} _TEMP_CONFIG_DIR="null" _SALTSTACK_REPO_URL="https://github.com/saltstack/salt.git" _SALT_REPO_URL=${_SALTSTACK_REPO_URL} -_DOWNSTREAM_PKG_REPO=$BS_FALSE _TEMP_KEYS_DIR="null" _SLEEP="${__DEFAULT_SLEEP}" _INSTALL_MASTER=$BS_FALSE @@ -311,21 +310,31 @@ __usage() { - onedir_rc Install latest onedir RC release. - onedir_rc [version] Install a specific version. Only supported for onedir RC packages available at repo.saltproject.io + - old-stable Install latest old stable release. + - old-stable [branch] Install latest version on a branch. Only supported + for packages available at repo.saltproject.io + - old-stable [version] Install a specific version. Only supported for + packages available at repo.saltproject.io + To pin a 3xxx minor version, specify it as 3xxx.0 Examples: - ${__ScriptName} - ${__ScriptName} stable - - ${__ScriptName} stable 2017.7 - - ${__ScriptName} stable 2017.7.2 + - ${__ScriptName} stable 3006 + - ${__ScriptName} stable 3006.1 - ${__ScriptName} testing - ${__ScriptName} git - ${__ScriptName} git 2017.7 - ${__ScriptName} git v2017.7.2 - ${__ScriptName} git 06f249901a2e2f1ed310d58ea3921a129f214358 - ${__ScriptName} onedir - - ${__ScriptName} onedir 3005 + - ${__ScriptName} onedir 3006 - ${__ScriptName} onedir_rc - - ${__ScriptName} onedir_rc 3005 + - ${__ScriptName} onedir_rc 3006 + - ${__ScriptName} old-stable + - ${__ScriptName} old-stable 3005 + - ${__ScriptName} old-stable 3005.1 + Options: -a Pip install all Python pkg dependencies for Salt. Requires -V to install @@ -401,9 +410,6 @@ __usage() { -v Display script version -V Install Salt into virtualenv (only available for Ubuntu based distributions) - -w Install packages from downstream package repository rather than - upstream, saltstack package repository. This is currently only - implemented for SUSE. -x Changes the Python version used to install Salt. For CentOS 6 git installations python2.7 is supported. Fedora git installation, CentOS 7, Ubuntu 18.04 support python3. @@ -420,7 +426,7 @@ EOT } # ---------- end of function __usage ---------- -while getopts ':hvnDc:g:Gyx:wk:s:MSNXCPFUKIA:i:Lp:dH:bflV:J:j:rR:aq' opt +while getopts ':hvnDc:g:Gyx:k:s:MSNXCPFUKIA:i:Lp:dH:bflV:J:j:rR:aq' opt do case "${opt}" in @@ -436,7 +442,6 @@ do echowarn "No need to provide this option anymore, now it is a default behavior." ;; - w ) _DOWNSTREAM_PKG_REPO=$BS_TRUE ;; k ) _TEMP_KEYS_DIR="$OPTARG" ;; s ) _SLEEP=$OPTARG ;; M ) _INSTALL_MASTER=$BS_TRUE ;; @@ -595,7 +600,7 @@ if [ "$#" -gt 0 ];then fi # Check installation type -if [ "$(echo "$ITYPE" | grep -E '(stable|testing|git|onedir|onedir_rc)')" = "" ]; then +if [ "$(echo "$ITYPE" | grep -E '(stable|testing|git|onedir|onedir_rc|old-stable)')" = "" ]; then echoerror "Installation type \"$ITYPE\" is not known..." exit 1 fi @@ -619,28 +624,41 @@ elif [ "$ITYPE" = "stable" ]; then _ONEDIR_REV="latest" ITYPE="onedir" else - if [ "$(echo "$1" | grep -E '^(nightly|latest|3006)$')" != "" ]; then + if [ "$(echo "$1" | grep -E '^(nightly|latest|3005|3006)$')" != "" ]; then ONEDIR_REV="$1" _ONEDIR_REV="$1" ITYPE="onedir" shift - elif [ "$(echo "$1" | grep -E '^(3003|3004|3005)$')" != "" ]; then - STABLE_REV="$1" - shift - elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{2}[5-9](\.[0-9]*)?)')" != "" ]; then ONEDIR_REV="minor/$1" _ONEDIR_REV="$1" ITYPE="onedir" shift + else + echo "Unknown stable version: $1 (valid: 3005, 3006, latest)" + exit 1 + fi + fi + +# If doing old-stable install, check if version specified +elif [ "$ITYPE" = "old-stable" ]; then + if [ "$#" -eq 0 ];then + ITYPE="stable" + else + if [ "$(echo "$1" | grep -E '^(3003|3004|3005)$')" != "" ]; then + STABLE_REV="$1" + ITYPE="stable" + shift elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{3}(\.[0-9]*)?)$')" != "" ]; then # Handle the 3xxx.0 version as 3xxx archive (pin to minor) and strip the fake ".0" suffix + ITYPE="stable" STABLE_REV=$(echo "$1" | sed -E 's/^([3-9][0-9]{3})\.0$/\1/') if [ "$(uname)" != "Darwin" ]; then STABLE_REV="archive/$STABLE_REV" fi shift else - echo "Unknown stable version: $1 (valid: 3003, 3004, 3005, 3006, latest)" + echo "Unknown old stable version: $1 (valid: 3003, 3004, 3005)" exit 1 fi fi @@ -4573,6 +4591,12 @@ install_fedora_onedir_post() { # CentOS Install Functions # __install_saltstack_rhel_repository() { + if [ "${DISTRO_MAJOR_VERSION}" -ge 9 ]; then + echoerror "Old stable repository unavailable on RH variants greater than or equal to 9" + echoerror "Use the stable install type." + exit 1 + fi + if [ "$ITYPE" = "stable" ]; then repo_rev="$STABLE_REV" else @@ -4827,7 +4851,6 @@ install_centos_git_deps() { # Set ONEDIR_REV to STABLE_REV in case we # end up calling install_centos_onedir_deps ONEDIR_REV=${STABLE_REV} - install_centos_stable_deps || \ install_centos_onedir_deps || \ return 1 @@ -7698,13 +7721,8 @@ __set_suse_pkg_repo() { DISTRO_REPO="SLE_${DISTRO_MAJOR_VERSION}_SP${SUSE_PATCHLEVEL}" fi - if [ "$_DOWNSTREAM_PKG_REPO" -eq $BS_TRUE ]; then - suse_pkg_url_base="https://download.opensuse.org/repositories/systemsmanagement:/saltstack" - suse_pkg_url_path="${DISTRO_REPO}/systemsmanagement:saltstack.repo" - else - suse_pkg_url_base="${HTTP_VAL}://repo.saltproject.io/opensuse" - suse_pkg_url_path="${DISTRO_REPO}/systemsmanagement:saltstack:products.repo" - fi + suse_pkg_url_base="https://download.opensuse.org/repositories/systemsmanagement:/saltstack" + suse_pkg_url_path="${DISTRO_REPO}/systemsmanagement:saltstack.repo" SUSE_PKG_URL="$suse_pkg_url_base/$suse_pkg_url_path" } @@ -7724,7 +7742,7 @@ __version_lte() { zypper --non-interactive install --auto-agree-with-licenses python || return 1 fi - if [ "$(python -c 'import sys; V1=tuple([int(i) for i in sys.argv[1].split(".")]); V2=tuple([int(i) for i in sys.argv[2].split(".")]); print V1<=V2' "$1" "$2")" = "True" ]; then + if [ "$(${_PY_EXE} -c 'import sys; V1=tuple([int(i) for i in sys.argv[1].split(".")]); V2=tuple([int(i) for i in sys.argv[2].split(".")]); print(V1<=V2)' "$1" "$2")" = "True" ]; then __ZYPPER_REQUIRES_REPLACE_FILES=${BS_TRUE} else __ZYPPER_REQUIRES_REPLACE_FILES=${BS_FALSE} @@ -8130,6 +8148,11 @@ install_opensuse_15_git() { return 0 } +install_opensuse_15_onedir_deps() { + __opensuse_prep_install || return 1 + return 0 +} + # # End of openSUSE Leap 15 # @@ -8159,6 +8182,13 @@ install_suse_15_git_deps() { return 0 } +install_suse_15_onedir_deps() { + __opensuse_prep_install || return 1 + install_opensuse_15_onedir_deps || return 1 + + return 0 +} + install_suse_15_stable() { install_opensuse_stable || return 1 return 0 @@ -8169,6 +8199,11 @@ install_suse_15_git() { return 0 } +install_suse_15_onedir() { + install_opensuse_stable || return 1 + return 0 +} + install_suse_15_stable_post() { install_opensuse_stable_post || return 1 return 0 @@ -8179,6 +8214,11 @@ install_suse_15_git_post() { return 0 } +install_suse_15_onedir_post() { + install_opensuse_stable_post || return 1 + return 0 +} + install_suse_15_restart_daemons() { install_opensuse_restart_daemons || return 1 return 0 @@ -8261,6 +8301,11 @@ install_suse_12_git_deps() { return 0 } +install_suse_12_onedir_deps() { + install_suse_12_stable_deps || return 1 + return 0 +} + install_suse_12_stable() { install_opensuse_stable || return 1 return 0 @@ -8271,6 +8316,11 @@ install_suse_12_git() { return 0 } +install_suse_12_onedir() { + install_opensuse_stable || return 1 + return 0 +} + install_suse_12_stable_post() { install_opensuse_stable_post || return 1 return 0 @@ -8281,6 +8331,11 @@ install_suse_12_git_post() { return 0 } +install_suse_12_onedir_post() { + install_opensuse_stable_post || return 1 + return 0 +} + install_suse_12_restart_daemons() { install_opensuse_restart_daemons || return 1 return 0 @@ -8357,6 +8412,11 @@ install_suse_11_git_deps() { return 0 } +install_suse_11_onedir_deps() { + install_suse_11_stable_deps || return 1 + return 0 +} + install_suse_11_stable() { install_opensuse_stable || return 1 return 0 @@ -8367,6 +8427,11 @@ install_suse_11_git() { return 0 } +install_suse_11_onedir() { + install_opensuse_stable || return 1 + return 0 +} + install_suse_11_stable_post() { install_opensuse_stable_post || return 1 return 0 @@ -8377,6 +8442,11 @@ install_suse_11_git_post() { return 0 } +install_suse_11_onedir_post() { + install_opensuse_stable_post || return 1 + return 0 +} + install_suse_11_restart_daemons() { install_opensuse_restart_daemons || return 1 return 0 From 3d097b8ed5579b3a94ccc3ba087b4d67d6d14fc3 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 4 Jul 2023 07:45:38 +0100 Subject: [PATCH 3/5] Upgrade to `cryptography==41.0.1`(and therefor `pyopenssl==23.2.0` due to https://github.com/advisories/GHSA-5cpq-8wj7-hf2v This only really impacts pip installs of Salt and the windows onedir since the linux and macos onedir build every package dependency from source, not from pre-existing wheels. Signed-off-by: Pedro Algarvio --- changelog/64595.security.md | 3 +++ requirements/darwin.txt | 2 +- requirements/static/ci/py3.10/cloud.txt | 4 ++-- requirements/static/ci/py3.10/darwin.txt | 4 ++-- requirements/static/ci/py3.10/freebsd.txt | 4 ++-- requirements/static/ci/py3.10/lint.txt | 4 ++-- requirements/static/ci/py3.10/linux.txt | 4 ++-- requirements/static/ci/py3.10/windows.txt | 4 ++-- requirements/static/ci/py3.7/cloud.txt | 4 ++-- requirements/static/ci/py3.7/freebsd.txt | 4 ++-- requirements/static/ci/py3.7/lint.txt | 4 ++-- requirements/static/ci/py3.7/linux.txt | 4 ++-- requirements/static/ci/py3.7/windows.txt | 4 ++-- requirements/static/ci/py3.8/cloud.txt | 4 ++-- requirements/static/ci/py3.8/freebsd.txt | 4 ++-- requirements/static/ci/py3.8/lint.txt | 4 ++-- requirements/static/ci/py3.8/linux.txt | 4 ++-- requirements/static/ci/py3.8/windows.txt | 4 ++-- requirements/static/ci/py3.9/cloud.txt | 4 ++-- requirements/static/ci/py3.9/darwin.txt | 4 ++-- requirements/static/ci/py3.9/freebsd.txt | 4 ++-- requirements/static/ci/py3.9/lint.txt | 4 ++-- requirements/static/ci/py3.9/linux.txt | 4 ++-- requirements/static/ci/py3.9/windows.txt | 4 ++-- requirements/static/pkg/linux.in | 2 +- requirements/static/pkg/py3.10/darwin.txt | 4 ++-- requirements/static/pkg/py3.10/freebsd.txt | 2 +- requirements/static/pkg/py3.10/linux.txt | 4 ++-- requirements/static/pkg/py3.10/windows.txt | 4 ++-- requirements/static/pkg/py3.7/freebsd.txt | 2 +- requirements/static/pkg/py3.7/linux.txt | 4 ++-- requirements/static/pkg/py3.7/windows.txt | 4 ++-- requirements/static/pkg/py3.8/freebsd.txt | 2 +- requirements/static/pkg/py3.8/linux.txt | 4 ++-- requirements/static/pkg/py3.8/windows.txt | 4 ++-- requirements/static/pkg/py3.9/darwin.txt | 4 ++-- requirements/static/pkg/py3.9/freebsd.txt | 2 +- requirements/static/pkg/py3.9/linux.txt | 4 ++-- requirements/static/pkg/py3.9/windows.txt | 4 ++-- requirements/windows.txt | 2 +- 40 files changed, 74 insertions(+), 71 deletions(-) create mode 100644 changelog/64595.security.md diff --git a/changelog/64595.security.md b/changelog/64595.security.md new file mode 100644 index 00000000000..63a7d529f92 --- /dev/null +++ b/changelog/64595.security.md @@ -0,0 +1,3 @@ +Upgrade to `cryptography==41.0.1`(and therefor `pyopenssl==23.2.0` due to https://github.com/advisories/GHSA-5cpq-8wj7-hf2v + +This only really impacts pip installs of Salt and the windows onedir since the linux and macos onedir build every package dependency from source, not from pre-existing wheels. diff --git a/requirements/darwin.txt b/requirements/darwin.txt index be8a547ae8e..a77dad5d7e3 100644 --- a/requirements/darwin.txt +++ b/requirements/darwin.txt @@ -5,7 +5,7 @@ apache-libcloud>=2.4.0 backports.ssl_match_hostname>=3.7.0.1; python_version < '3.7' cherrypy>=17.4.1 -cryptography>=39.0.1 +cryptography>=41.0.1 gitpython>=3.1.30; python_version >= '3.7' idna>=2.8 linode-python>=1.1.1 diff --git a/requirements/static/ci/py3.10/cloud.txt b/requirements/static/ci/py3.10/cloud.txt index daf4d5e1570..9da18084d70 100644 --- a/requirements/static/ci/py3.10/cloud.txt +++ b/requirements/static/ci/py3.10/cloud.txt @@ -385,7 +385,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==1.0.15 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/static/pkg/linux.in # adal @@ -669,7 +669,7 @@ pyjwt==2.4.0 # via adal pynacl==1.4.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/static/pkg/linux.in # etcd3-py diff --git a/requirements/static/ci/py3.10/darwin.txt b/requirements/static/ci/py3.10/darwin.txt index 800698143fa..8d4edd7d491 100644 --- a/requirements/static/ci/py3.10/darwin.txt +++ b/requirements/static/ci/py3.10/darwin.txt @@ -386,7 +386,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==0.3.29 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/darwin.txt # adal @@ -670,7 +670,7 @@ pyjwt==2.4.0 # via adal pynacl==1.3.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/darwin.txt # etcd3-py diff --git a/requirements/static/ci/py3.10/freebsd.txt b/requirements/static/ci/py3.10/freebsd.txt index 9f485ddbca5..92e27ed6d94 100644 --- a/requirements/static/ci/py3.10/freebsd.txt +++ b/requirements/static/ci/py3.10/freebsd.txt @@ -384,7 +384,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==0.3.29 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.0 # via # adal # azure-cosmosdb-table @@ -669,7 +669,7 @@ pyjwt==2.4.0 # via adal pynacl==1.3.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/static/pkg/freebsd.in # etcd3-py diff --git a/requirements/static/ci/py3.10/lint.txt b/requirements/static/ci/py3.10/lint.txt index 72daefca405..9b35d0508bb 100644 --- a/requirements/static/ci/py3.10/lint.txt +++ b/requirements/static/ci/py3.10/lint.txt @@ -390,7 +390,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==1.0.15 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/static/pkg/linux.in # adal @@ -675,7 +675,7 @@ pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.4.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/static/pkg/linux.in # etcd3-py diff --git a/requirements/static/ci/py3.10/linux.txt b/requirements/static/ci/py3.10/linux.txt index 94a3863c683..6c88290286c 100644 --- a/requirements/static/ci/py3.10/linux.txt +++ b/requirements/static/ci/py3.10/linux.txt @@ -398,7 +398,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==0.3.29 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/static/pkg/linux.in # adal @@ -689,7 +689,7 @@ pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/static/pkg/linux.in # etcd3-py diff --git a/requirements/static/ci/py3.10/windows.txt b/requirements/static/ci/py3.10/windows.txt index 091c51ec590..033084010b5 100644 --- a/requirements/static/ci/py3.10/windows.txt +++ b/requirements/static/ci/py3.10/windows.txt @@ -71,7 +71,7 @@ colorama==0.4.1 # via pytest contextvars==2.4 # via -r requirements/base.txt -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/windows.txt # etcd3-py @@ -237,7 +237,7 @@ pymssql==2.2.7 # via -r requirements/windows.txt pymysql==1.0.2 # via -r requirements/windows.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/windows.txt # etcd3-py diff --git a/requirements/static/ci/py3.7/cloud.txt b/requirements/static/ci/py3.7/cloud.txt index 74e67f129c1..7ab45ff74f7 100644 --- a/requirements/static/ci/py3.7/cloud.txt +++ b/requirements/static/ci/py3.7/cloud.txt @@ -392,7 +392,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==1.0.15 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/static/pkg/linux.in # adal @@ -711,7 +711,7 @@ pyjwt==2.4.0 # via adal pynacl==1.4.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/static/pkg/linux.in # etcd3-py diff --git a/requirements/static/ci/py3.7/freebsd.txt b/requirements/static/ci/py3.7/freebsd.txt index 661a1670dde..13e97515d7d 100644 --- a/requirements/static/ci/py3.7/freebsd.txt +++ b/requirements/static/ci/py3.7/freebsd.txt @@ -391,7 +391,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==0.3.29 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.0 # via # adal # azure-cosmosdb-table @@ -705,7 +705,7 @@ pyjwt==2.4.0 # via adal pynacl==1.3.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/static/pkg/freebsd.in # etcd3-py diff --git a/requirements/static/ci/py3.7/lint.txt b/requirements/static/ci/py3.7/lint.txt index 8d8771573c6..33080ce341d 100644 --- a/requirements/static/ci/py3.7/lint.txt +++ b/requirements/static/ci/py3.7/lint.txt @@ -399,7 +399,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==1.0.15 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/static/pkg/linux.in # adal @@ -718,7 +718,7 @@ pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.4.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/static/pkg/linux.in # etcd3-py diff --git a/requirements/static/ci/py3.7/linux.txt b/requirements/static/ci/py3.7/linux.txt index 5f0a0509d76..c8307fc7986 100644 --- a/requirements/static/ci/py3.7/linux.txt +++ b/requirements/static/ci/py3.7/linux.txt @@ -405,7 +405,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==0.3.29 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/static/pkg/linux.in # adal @@ -725,7 +725,7 @@ pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/static/pkg/linux.in # etcd3-py diff --git a/requirements/static/ci/py3.7/windows.txt b/requirements/static/ci/py3.7/windows.txt index 247778c256e..37aaab11ea7 100644 --- a/requirements/static/ci/py3.7/windows.txt +++ b/requirements/static/ci/py3.7/windows.txt @@ -77,7 +77,7 @@ colorama==0.4.1 # via pytest contextvars==2.4 # via -r requirements/base.txt -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/windows.txt # etcd3-py @@ -250,7 +250,7 @@ pymssql==2.2.1 # via -r requirements/windows.txt pymysql==1.0.2 # via -r requirements/windows.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/windows.txt # etcd3-py diff --git a/requirements/static/ci/py3.8/cloud.txt b/requirements/static/ci/py3.8/cloud.txt index fe4a7bcea2b..b60fd1d789d 100644 --- a/requirements/static/ci/py3.8/cloud.txt +++ b/requirements/static/ci/py3.8/cloud.txt @@ -390,7 +390,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==1.0.15 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/static/pkg/linux.in # adal @@ -700,7 +700,7 @@ pyjwt==2.4.0 # via adal pynacl==1.4.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/static/pkg/linux.in # etcd3-py diff --git a/requirements/static/ci/py3.8/freebsd.txt b/requirements/static/ci/py3.8/freebsd.txt index 7b8109b51eb..a38caa62b63 100644 --- a/requirements/static/ci/py3.8/freebsd.txt +++ b/requirements/static/ci/py3.8/freebsd.txt @@ -389,7 +389,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==0.3.29 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.0 # via # adal # azure-cosmosdb-table @@ -695,7 +695,7 @@ pyjwt==2.4.0 # via adal pynacl==1.3.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/static/pkg/freebsd.in # etcd3-py diff --git a/requirements/static/ci/py3.8/lint.txt b/requirements/static/ci/py3.8/lint.txt index b0a9aa31f4a..c9084b3f32a 100644 --- a/requirements/static/ci/py3.8/lint.txt +++ b/requirements/static/ci/py3.8/lint.txt @@ -397,7 +397,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==1.0.15 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/static/pkg/linux.in # adal @@ -709,7 +709,7 @@ pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.4.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/static/pkg/linux.in # etcd3-py diff --git a/requirements/static/ci/py3.8/linux.txt b/requirements/static/ci/py3.8/linux.txt index 19c4a389755..88bdfd4febf 100644 --- a/requirements/static/ci/py3.8/linux.txt +++ b/requirements/static/ci/py3.8/linux.txt @@ -403,7 +403,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==0.3.29 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/static/pkg/linux.in # adal @@ -715,7 +715,7 @@ pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/static/pkg/linux.in # etcd3-py diff --git a/requirements/static/ci/py3.8/windows.txt b/requirements/static/ci/py3.8/windows.txt index deb490a493b..0638887da49 100644 --- a/requirements/static/ci/py3.8/windows.txt +++ b/requirements/static/ci/py3.8/windows.txt @@ -73,7 +73,7 @@ colorama==0.4.1 # via pytest contextvars==2.4 # via -r requirements/base.txt -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/windows.txt # etcd3-py @@ -238,7 +238,7 @@ pymssql==2.2.1 # via -r requirements/windows.txt pymysql==1.0.2 # via -r requirements/windows.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/windows.txt # etcd3-py diff --git a/requirements/static/ci/py3.9/cloud.txt b/requirements/static/ci/py3.9/cloud.txt index ec7cb3da6ba..a866a0b15f6 100644 --- a/requirements/static/ci/py3.9/cloud.txt +++ b/requirements/static/ci/py3.9/cloud.txt @@ -390,7 +390,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==1.0.15 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/static/pkg/linux.in # adal @@ -703,7 +703,7 @@ pyjwt==2.4.0 # via adal pynacl==1.4.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/static/pkg/linux.in # etcd3-py diff --git a/requirements/static/ci/py3.9/darwin.txt b/requirements/static/ci/py3.9/darwin.txt index 2101a94d609..a309cf7c1ac 100644 --- a/requirements/static/ci/py3.9/darwin.txt +++ b/requirements/static/ci/py3.9/darwin.txt @@ -391,7 +391,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==0.3.29 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/darwin.txt # adal @@ -699,7 +699,7 @@ pyjwt==2.4.0 # via adal pynacl==1.3.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/darwin.txt # etcd3-py diff --git a/requirements/static/ci/py3.9/freebsd.txt b/requirements/static/ci/py3.9/freebsd.txt index e7d17091d8d..b352259132b 100644 --- a/requirements/static/ci/py3.9/freebsd.txt +++ b/requirements/static/ci/py3.9/freebsd.txt @@ -389,7 +389,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==0.3.29 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.0 # via # adal # azure-cosmosdb-table @@ -698,7 +698,7 @@ pyjwt==2.4.0 # via adal pynacl==1.3.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/static/pkg/freebsd.in # etcd3-py diff --git a/requirements/static/ci/py3.9/lint.txt b/requirements/static/ci/py3.9/lint.txt index 50e4c72ed5e..ca6f7af483d 100644 --- a/requirements/static/ci/py3.9/lint.txt +++ b/requirements/static/ci/py3.9/lint.txt @@ -395,7 +395,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==1.0.15 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/static/pkg/linux.in # adal @@ -710,7 +710,7 @@ pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.4.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/static/pkg/linux.in # etcd3-py diff --git a/requirements/static/ci/py3.9/linux.txt b/requirements/static/ci/py3.9/linux.txt index 647f4d91017..fd99af75b6a 100644 --- a/requirements/static/ci/py3.9/linux.txt +++ b/requirements/static/ci/py3.9/linux.txt @@ -405,7 +405,7 @@ contextvars==2.4 # via -r requirements/base.txt croniter==0.3.29 ; sys_platform != "win32" # via -r requirements/static/ci/common.in -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/static/pkg/linux.in # adal @@ -720,7 +720,7 @@ pymysql==1.0.2 ; python_version > "3.5" # via -r requirements/static/ci/linux.in pynacl==1.3.0 # via paramiko -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/static/pkg/linux.in # etcd3-py diff --git a/requirements/static/ci/py3.9/windows.txt b/requirements/static/ci/py3.9/windows.txt index 0f62ef8482d..671f49ddcc4 100644 --- a/requirements/static/ci/py3.9/windows.txt +++ b/requirements/static/ci/py3.9/windows.txt @@ -73,7 +73,7 @@ colorama==0.4.1 # via pytest contextvars==2.4 # via -r requirements/base.txt -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/windows.txt # etcd3-py @@ -239,7 +239,7 @@ pymssql==2.2.1 # via -r requirements/windows.txt pymysql==1.0.2 # via -r requirements/windows.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via # -r requirements/windows.txt # etcd3-py diff --git a/requirements/static/pkg/linux.in b/requirements/static/pkg/linux.in index 3555af3b9c3..340dde10eaa 100644 --- a/requirements/static/pkg/linux.in +++ b/requirements/static/pkg/linux.in @@ -10,4 +10,4 @@ rpm-vercmp setproctitle>=1.2.3 timelib>=0.2.5 importlib-metadata>=3.3.0 -cryptography>=39.0.1 +cryptography>=41.0.1 diff --git a/requirements/static/pkg/py3.10/darwin.txt b/requirements/static/pkg/py3.10/darwin.txt index 276394f1f8d..77f7ac11e69 100644 --- a/requirements/static/pkg/py3.10/darwin.txt +++ b/requirements/static/pkg/py3.10/darwin.txt @@ -18,7 +18,7 @@ cherrypy==18.6.1 # via -r requirements/darwin.txt contextvars==2.4 # via -r requirements/base.txt -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/darwin.txt # pyopenssl @@ -81,7 +81,7 @@ pycparser==2.21 # cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via -r requirements/darwin.txt python-dateutil==2.8.0 # via -r requirements/darwin.txt diff --git a/requirements/static/pkg/py3.10/freebsd.txt b/requirements/static/pkg/py3.10/freebsd.txt index fd41185e664..eb3f047d200 100644 --- a/requirements/static/pkg/py3.10/freebsd.txt +++ b/requirements/static/pkg/py3.10/freebsd.txt @@ -69,7 +69,7 @@ pycparser==2.21 ; python_version >= "3.9" # cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via -r requirements/static/pkg/freebsd.in python-dateutil==2.8.1 # via -r requirements/static/pkg/freebsd.in diff --git a/requirements/static/pkg/py3.10/linux.txt b/requirements/static/pkg/py3.10/linux.txt index 18d7c07ca8f..e9f1c6c50e0 100644 --- a/requirements/static/pkg/py3.10/linux.txt +++ b/requirements/static/pkg/py3.10/linux.txt @@ -16,7 +16,7 @@ cherrypy==18.6.1 # via -r requirements/static/pkg/linux.in contextvars==2.4 # via -r requirements/base.txt -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/static/pkg/linux.in # pyopenssl @@ -69,7 +69,7 @@ pycparser==2.21 ; python_version >= "3.9" # cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via -r requirements/static/pkg/linux.in python-dateutil==2.8.1 # via -r requirements/static/pkg/linux.in diff --git a/requirements/static/pkg/py3.10/windows.txt b/requirements/static/pkg/py3.10/windows.txt index f619ededaa4..72d665f7fa6 100644 --- a/requirements/static/pkg/py3.10/windows.txt +++ b/requirements/static/pkg/py3.10/windows.txt @@ -23,7 +23,7 @@ clr-loader==0.2.4 # via pythonnet contextvars==2.4 # via -r requirements/base.txt -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/windows.txt # pyopenssl @@ -90,7 +90,7 @@ pymssql==2.2.7 # via -r requirements/windows.txt pymysql==1.0.2 # via -r requirements/windows.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via -r requirements/windows.txt python-dateutil==2.8.1 # via -r requirements/windows.txt diff --git a/requirements/static/pkg/py3.7/freebsd.txt b/requirements/static/pkg/py3.7/freebsd.txt index 15b2d47cca7..ae5865986b3 100644 --- a/requirements/static/pkg/py3.7/freebsd.txt +++ b/requirements/static/pkg/py3.7/freebsd.txt @@ -67,7 +67,7 @@ pycparser==2.17 # via cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via -r requirements/static/pkg/freebsd.in python-dateutil==2.8.1 # via -r requirements/static/pkg/freebsd.in diff --git a/requirements/static/pkg/py3.7/linux.txt b/requirements/static/pkg/py3.7/linux.txt index f44232437ad..d1c058f6f48 100644 --- a/requirements/static/pkg/py3.7/linux.txt +++ b/requirements/static/pkg/py3.7/linux.txt @@ -16,7 +16,7 @@ cherrypy==18.6.1 # via -r requirements/static/pkg/linux.in contextvars==2.4 # via -r requirements/base.txt -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/static/pkg/linux.in # pyopenssl @@ -67,7 +67,7 @@ pycparser==2.17 # via cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via -r requirements/static/pkg/linux.in python-dateutil==2.8.1 # via -r requirements/static/pkg/linux.in diff --git a/requirements/static/pkg/py3.7/windows.txt b/requirements/static/pkg/py3.7/windows.txt index ec7a648807b..23b16d98e78 100644 --- a/requirements/static/pkg/py3.7/windows.txt +++ b/requirements/static/pkg/py3.7/windows.txt @@ -23,7 +23,7 @@ clr-loader==0.2.4 # via pythonnet contextvars==2.4 # via -r requirements/base.txt -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/windows.txt # pyopenssl @@ -90,7 +90,7 @@ pymssql==2.2.1 # via -r requirements/windows.txt pymysql==1.0.2 # via -r requirements/windows.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via -r requirements/windows.txt python-dateutil==2.8.1 # via -r requirements/windows.txt diff --git a/requirements/static/pkg/py3.8/freebsd.txt b/requirements/static/pkg/py3.8/freebsd.txt index ef5b28be524..2419a87646c 100644 --- a/requirements/static/pkg/py3.8/freebsd.txt +++ b/requirements/static/pkg/py3.8/freebsd.txt @@ -67,7 +67,7 @@ pycparser==2.17 # via cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via -r requirements/static/pkg/freebsd.in python-dateutil==2.8.1 # via -r requirements/static/pkg/freebsd.in diff --git a/requirements/static/pkg/py3.8/linux.txt b/requirements/static/pkg/py3.8/linux.txt index 1db80471daf..6dc2d7918d9 100644 --- a/requirements/static/pkg/py3.8/linux.txt +++ b/requirements/static/pkg/py3.8/linux.txt @@ -16,7 +16,7 @@ cherrypy==18.6.1 # via -r requirements/static/pkg/linux.in contextvars==2.4 # via -r requirements/base.txt -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/static/pkg/linux.in # pyopenssl @@ -67,7 +67,7 @@ pycparser==2.17 # via cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via -r requirements/static/pkg/linux.in python-dateutil==2.8.1 # via -r requirements/static/pkg/linux.in diff --git a/requirements/static/pkg/py3.8/windows.txt b/requirements/static/pkg/py3.8/windows.txt index b37c1ed5819..0363e776489 100644 --- a/requirements/static/pkg/py3.8/windows.txt +++ b/requirements/static/pkg/py3.8/windows.txt @@ -23,7 +23,7 @@ clr-loader==0.2.4 # via pythonnet contextvars==2.4 # via -r requirements/base.txt -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/windows.txt # pyopenssl @@ -90,7 +90,7 @@ pymssql==2.2.1 # via -r requirements/windows.txt pymysql==1.0.2 # via -r requirements/windows.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via -r requirements/windows.txt python-dateutil==2.8.1 # via -r requirements/windows.txt diff --git a/requirements/static/pkg/py3.9/darwin.txt b/requirements/static/pkg/py3.9/darwin.txt index 636e8766bff..6dc90cc344f 100644 --- a/requirements/static/pkg/py3.9/darwin.txt +++ b/requirements/static/pkg/py3.9/darwin.txt @@ -18,7 +18,7 @@ cherrypy==18.6.1 # via -r requirements/darwin.txt contextvars==2.4 # via -r requirements/base.txt -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/darwin.txt # pyopenssl @@ -81,7 +81,7 @@ pycparser==2.21 # cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via -r requirements/darwin.txt python-dateutil==2.8.0 # via -r requirements/darwin.txt diff --git a/requirements/static/pkg/py3.9/freebsd.txt b/requirements/static/pkg/py3.9/freebsd.txt index c8e2570bf9d..dfa83b9c31d 100644 --- a/requirements/static/pkg/py3.9/freebsd.txt +++ b/requirements/static/pkg/py3.9/freebsd.txt @@ -69,7 +69,7 @@ pycparser==2.21 ; python_version >= "3.9" # cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via -r requirements/static/pkg/freebsd.in python-dateutil==2.8.1 # via -r requirements/static/pkg/freebsd.in diff --git a/requirements/static/pkg/py3.9/linux.txt b/requirements/static/pkg/py3.9/linux.txt index 0c366bf1869..56e78f78351 100644 --- a/requirements/static/pkg/py3.9/linux.txt +++ b/requirements/static/pkg/py3.9/linux.txt @@ -16,7 +16,7 @@ cherrypy==18.6.1 # via -r requirements/static/pkg/linux.in contextvars==2.4 # via -r requirements/base.txt -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/static/pkg/linux.in # pyopenssl @@ -69,7 +69,7 @@ pycparser==2.21 ; python_version >= "3.9" # cffi pycryptodomex==3.9.8 # via -r requirements/crypto.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via -r requirements/static/pkg/linux.in python-dateutil==2.8.1 # via -r requirements/static/pkg/linux.in diff --git a/requirements/static/pkg/py3.9/windows.txt b/requirements/static/pkg/py3.9/windows.txt index f9a874bd592..273acb031d2 100644 --- a/requirements/static/pkg/py3.9/windows.txt +++ b/requirements/static/pkg/py3.9/windows.txt @@ -23,7 +23,7 @@ clr-loader==0.2.4 # via pythonnet contextvars==2.4 # via -r requirements/base.txt -cryptography==39.0.2 +cryptography==41.0.1 # via # -r requirements/windows.txt # pyopenssl @@ -90,7 +90,7 @@ pymssql==2.2.1 # via -r requirements/windows.txt pymysql==1.0.2 # via -r requirements/windows.txt -pyopenssl==23.0.0 +pyopenssl==23.2.0 # via -r requirements/windows.txt python-dateutil==2.8.1 # via -r requirements/windows.txt diff --git a/requirements/windows.txt b/requirements/windows.txt index dacdbd1279a..b91cd0ecf2d 100644 --- a/requirements/windows.txt +++ b/requirements/windows.txt @@ -10,7 +10,7 @@ backports.ssl-match-hostname>=3.7.0.1; python_version < '3.7' certifi>=2022.12.07 cffi>=1.14.5 cherrypy>=18.6.1 -cryptography>=39.0.1 +cryptography>=41.0.1 gitpython>=3.1.30; python_version >= '3.7' ioloop>=0.1a0 lxml>=4.6.3 From dc9c22ab9b76b5d444a279e130c8c6fee9bc7500 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 5 Jul 2023 09:37:12 +0100 Subject: [PATCH 4/5] Switch to Debian 12 to build packages. The latest cryptography 1.41.1 requires a version of rustc which is not available on Debian 11 Signed-off-by: Pedro Algarvio --- .../setup-python-tools-scripts/action.yml | 7 ++- .github/workflows/build-deb-packages.yml | 2 +- .pre-commit-config.yaml | 11 ++++ requirements/static/ci/py3.11/tools.txt | 58 +++++++++++++++++++ 4 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 requirements/static/ci/py3.11/tools.txt diff --git a/.github/actions/setup-python-tools-scripts/action.yml b/.github/actions/setup-python-tools-scripts/action.yml index dcd46feb2b0..72bcf3b1d37 100644 --- a/.github/actions/setup-python-tools-scripts/action.yml +++ b/.github/actions/setup-python-tools-scripts/action.yml @@ -33,7 +33,12 @@ runs: shell: bash working-directory: ${{ inputs.cwd }} run: | - python3 -m pip install -r requirements/static/ci/py${{ steps.get-python-version.outputs.version }}/tools.txt + (python3 -m pip install --help | grep break-system-packages > /dev/null 2>&1) && exitcode=0 || exitcode=1 + if [ $exitcode -eq 0 ]; then + python3 -m pip install --break-system-packages -r requirements/static/ci/py${{ steps.get-python-version.outputs.version }}/tools.txt + else + python3 -m pip install -r requirements/static/ci/py${{ steps.get-python-version.outputs.version }}/tools.txt + fi - name: Get 'python-tools-scripts' Version id: get-version diff --git a/.github/workflows/build-deb-packages.yml b/.github/workflows/build-deb-packages.yml index c24fef37c14..42f7f4eb6e7 100644 --- a/.github/workflows/build-deb-packages.yml +++ b/.github/workflows/build-deb-packages.yml @@ -35,7 +35,7 @@ jobs: - src container: - image: ghcr.io/saltstack/salt-ci-containers/packaging:debian-11 + image: ghcr.io/saltstack/salt-ci-containers/packaging:debian-12 steps: # Checkout here so we can easily use custom actions diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3bf36c32476..d6cb848d7e6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1022,6 +1022,17 @@ repos: - --py-version=3.10 - --pip-args=--constraint=requirements/static/ci/py{py_version}/linux.txt - requirements/static/ci/tools.in + + - id: pip-tools-compile + alias: compile-ci-tools-3.11-requirements + name: Linux CI Py3.11 Tools Requirements + files: ^requirements/static/ci/(tools\.in|py3.11/(tools|linux)\.txt)$ + pass_filenames: false + args: + - -v + - --py-version=3.11 + - --pip-args=--constraint=requirements/static/ci/py{py_version}/linux.txt + - requirements/static/ci/tools.in # <---- Tools ----------------------------------------------------------------------------------------------------- # ----- Code Formatting -------------------------------------------------------------------------------------------> diff --git a/requirements/static/ci/py3.11/tools.txt b/requirements/static/ci/py3.11/tools.txt new file mode 100644 index 00000000000..bdd05d1f2ee --- /dev/null +++ b/requirements/static/ci/py3.11/tools.txt @@ -0,0 +1,58 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements/static/ci/py3.11/tools.txt --pip-args='--constraint=requirements/static/ci/py3.11/linux.txt' requirements/static/ci/tools.in +# +attrs==22.1.0 + # via + # -r requirements/static/ci/tools.in + # python-tools-scripts +boto3==1.21.46 + # via -r requirements/static/ci/tools.in +botocore==1.24.46 + # via + # boto3 + # s3transfer +certifi==2022.12.7 + # via requests +charset-normalizer==3.0.1 + # via requests +commonmark==0.9.1 + # via rich +idna==3.4 + # via requests +jinja2==3.1.2 + # via -r requirements/static/ci/tools.in +jmespath==1.0.1 + # via + # boto3 + # botocore +markupsafe==2.1.2 + # via jinja2 +packaging==23.0 + # via -r requirements/static/ci/tools.in +pygments==2.13.0 + # via rich +python-dateutil==2.8.2 + # via botocore +python-tools-scripts==0.12.0 + # via -r requirements/static/ci/tools.in +pyyaml==6.0 + # via -r requirements/static/ci/tools.in +requests==2.31.0 + # via + # python-tools-scripts + # virustotal3 +rich==12.5.1 + # via python-tools-scripts +s3transfer==0.5.2 + # via boto3 +six==1.16.0 + # via python-dateutil +urllib3==1.26.12 + # via + # botocore + # requests +virustotal3==1.0.8 + # via -r requirements/static/ci/tools.in From 7a2eeada097b4f041cd305cddd959cc39ac1c2a2 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 7 Jul 2023 07:52:37 +0100 Subject: [PATCH 5/5] Allow forcing the package tests to run by the `test:pkg` label Signed-off-by: Pedro Algarvio --- tools/ci.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/tools/ci.py b/tools/ci.py index 566a1f388f6..78e65ad84c3 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -305,6 +305,23 @@ def define_jobs( ) return + # This is a pull-request + + labels: list[str] = [] + gh_event_path = os.environ.get("GITHUB_EVENT_PATH") or None + if gh_event_path is not None: + try: + gh_event = json.loads(open(gh_event_path).read()) + except Exception as exc: + ctx.error( + f"Could not load the GH Event payload from {gh_event_path!r}:\n", exc + ) + ctx.exit(1) + + labels.extend( + label[0] for label in _get_pr_test_labels_from_event_payload(gh_event) + ) + if not changed_files.exists(): ctx.error(f"The '{changed_files}' file does not exist.") ctx.error( @@ -354,9 +371,16 @@ def define_jobs( changed_files_contents["workflows"], } if jobs["test-pkg"] and required_pkg_test_changes == {"false"}: - with open(github_step_summary, "a", encoding="utf-8") as wfh: - wfh.write("De-selecting the 'test-pkg' job.\n") - jobs["test-pkg"] = False + if "test:pkg" in labels: + with open(github_step_summary, "a", encoding="utf-8") as wfh: + wfh.write( + "The 'test-pkg' job is forcefully selected by the use of the 'test:pkg' label.\n" + ) + jobs["test-pkg"] = True + else: + with open(github_step_summary, "a", encoding="utf-8") as wfh: + wfh.write("De-selecting the 'test-pkg' job.\n") + jobs["test-pkg"] = False if jobs["test-pkg-download"] and required_pkg_test_changes == {"false"}: with open(github_step_summary, "a", encoding="utf-8") as wfh: