Use importlib.metadata for python >= 3.10

Add slots to DependencyGraph

Pinned networkx to 3.1

Pinned networkx to latest version supported on python 3.8 to support running salt-ssh to systems with python 3.8

Fixed assignment of env in support helpers VirtualEnv

Restrict networkx version with contraints instead of in base

Fixed setting context retcode in some places where errors are returned.
Removed unnecessary pylint ignore comment.
Reverted importlib-metadata version bump.

Added networkx to versions report

Actually fixed VirtualEnv env assignment from kwargs
This commit is contained in:
bdrx312 2024-10-03 23:47:43 -04:00 committed by Daniel Wozniak
parent 66caa58346
commit 221420cd57
51 changed files with 149 additions and 131 deletions

View file

@ -5,6 +5,7 @@ jmespath
msgpack>=1.0.0
PyYAML
MarkupSafe
# pin to a version available on all supported python versions so salt-ssh can run on older targets
networkx
requests>=2.31.0 ; python_version < '3.8'
requests>=2.32.0 ; python_version >= '3.8'
@ -26,7 +27,7 @@ pyopenssl>=24.0.0
python-dateutil>=2.8.1
python-gnupg>=0.4.7
cherrypy>=18.6.1
importlib-metadata>=4.3.0
importlib-metadata>=3.3.0
cryptography>=42.0.0
# From old requirements/static/pkg/linux.in

View file

@ -1,3 +1,5 @@
setuptools >= 65.6.3,< 69.0
setuptools-scm < 8.0.0
pip >= 23.3,< 24.0
# Restrict to a version that works on all supported python versions so salt-ssh can run on older targets
networkx >= 3.0,< 3.2

View file

@ -282,8 +282,9 @@ netmiko==4.2.0
# via napalm
netutils==1.6.0
# via napalm
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.10/darwin.txt
# -r requirements/base.txt
ntc-templates==4.0.1

View file

@ -285,8 +285,9 @@ netmiko==4.2.0
# via napalm
netutils==1.6.0
# via napalm
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.10/freebsd.txt
# -r requirements/base.txt
ntc-templates==4.0.1

View file

@ -310,8 +310,9 @@ netmiko==4.2.0
# via napalm
netutils==1.6.0
# via napalm
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.10/linux.txt
# -r requirements/base.txt
ntc-templates==4.0.1

View file

@ -253,8 +253,9 @@ multidict==6.0.4
# -c requirements/static/ci/../pkg/py3.10/windows.txt
# aiohttp
# yarl
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.10/windows.txt
# -r requirements/base.txt
packaging==23.1

View file

@ -275,8 +275,9 @@ netmiko==4.2.0
# via napalm
netutils==1.6.0
# via napalm
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.11/darwin.txt
# -r requirements/base.txt
ntc-templates==4.0.1

View file

@ -278,8 +278,9 @@ netmiko==4.2.0
# via napalm
netutils==1.6.0
# via napalm
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.11/freebsd.txt
# -r requirements/base.txt
ntc-templates==4.0.1

View file

@ -301,8 +301,9 @@ netmiko==4.2.0
# via napalm
netutils==1.6.0
# via napalm
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.11/linux.txt
# -r requirements/base.txt
ntc-templates==4.0.1

View file

@ -246,8 +246,9 @@ multidict==6.0.4
# -c requirements/static/ci/../pkg/py3.11/windows.txt
# aiohttp
# yarl
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.11/windows.txt
# -r requirements/base.txt
packaging==23.1

View file

@ -385,8 +385,9 @@ netutils==1.6.0
# via
# -c requirements/static/ci/py3.12/linux.txt
# napalm
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.12/linux.txt
# -c requirements/static/ci/py3.12/linux.txt
# -r requirements/base.txt

View file

@ -275,8 +275,9 @@ netmiko==4.2.0
# via napalm
netutils==1.6.0
# via napalm
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.12/darwin.txt
# -r requirements/base.txt
ntc-templates==4.0.1

View file

@ -158,8 +158,9 @@ multidict==6.0.4
# yarl
myst-docutils[linkify]==1.0.0
# via -r requirements/static/ci/docs.in
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/py3.12/linux.txt
# -r requirements/base.txt
packaging==23.1

View file

@ -278,8 +278,9 @@ netmiko==4.2.0
# via napalm
netutils==1.6.0
# via napalm
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.12/freebsd.txt
# -r requirements/base.txt
ntc-templates==4.0.1

View file

@ -412,8 +412,9 @@ netutils==1.6.0
# via
# -c requirements/static/ci/py3.12/linux.txt
# napalm
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.12/linux.txt
# -c requirements/static/ci/py3.12/linux.txt
# -r requirements/base.txt

View file

@ -301,8 +301,9 @@ netmiko==4.2.0
# via napalm
netutils==1.6.0
# via napalm
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.12/linux.txt
# -r requirements/base.txt
ntc-templates==4.0.1

View file

@ -246,8 +246,9 @@ multidict==6.0.4
# -c requirements/static/ci/../pkg/py3.12/windows.txt
# aiohttp
# yarl
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.12/windows.txt
# -r requirements/base.txt
packaging==23.1

View file

@ -291,6 +291,7 @@ netutils==1.6.0
# via napalm
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.8/freebsd.txt
# -r requirements/base.txt
ntc-templates==4.0.1

View file

@ -310,6 +310,7 @@ netutils==1.6.0
# via napalm
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.8/linux.txt
# -r requirements/base.txt
ntc-templates==4.0.1

View file

@ -259,6 +259,7 @@ multidict==6.0.4
# yarl
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.8/windows.txt
# -r requirements/base.txt
packaging==23.1

View file

@ -282,8 +282,9 @@ netmiko==4.2.0
# via napalm
netutils==1.6.0
# via napalm
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.9/darwin.txt
# -r requirements/base.txt
ntc-templates==4.0.1

View file

@ -285,8 +285,9 @@ netmiko==4.2.0
# via napalm
netutils==1.6.0
# via napalm
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.9/freebsd.txt
# -r requirements/base.txt
ntc-templates==4.0.1

View file

@ -304,8 +304,9 @@ netmiko==4.2.0
# via napalm
netutils==1.6.0
# via napalm
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.9/linux.txt
# -r requirements/base.txt
ntc-templates==4.0.1

View file

@ -253,8 +253,9 @@ multidict==6.0.4
# -c requirements/static/ci/../pkg/py3.9/windows.txt
# aiohttp
# yarl
networkx==3.2.1
networkx==3.1
# via
# -c requirements/constraints.txt
# -c requirements/static/ci/../pkg/py3.9/windows.txt
# -r requirements/base.txt
packaging==23.1

View file

@ -83,8 +83,10 @@ multidict==6.0.4
# via
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements/base.txt
networkx==3.1
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -83,8 +83,10 @@ multidict==6.0.4
# via
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements/base.txt
networkx==3.1
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -83,8 +83,10 @@ multidict==6.0.4
# via
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements/base.txt
networkx==3.1
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -91,8 +91,10 @@ multidict==6.0.4
# via
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements/base.txt
networkx==3.1
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -81,8 +81,10 @@ multidict==6.0.4
# via
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements/base.txt
networkx==3.1
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -81,8 +81,10 @@ multidict==6.0.4
# via
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements/base.txt
networkx==3.1
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -81,8 +81,10 @@ multidict==6.0.4
# via
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements/base.txt
networkx==3.1
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -89,8 +89,10 @@ multidict==6.0.4
# via
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements/base.txt
networkx==3.1
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -81,8 +81,10 @@ multidict==6.0.4
# via
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements/base.txt
networkx==3.1
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -81,8 +81,10 @@ multidict==6.0.4
# via
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements/base.txt
networkx==3.1
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -81,8 +81,10 @@ multidict==6.0.4
# via
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements/base.txt
networkx==3.1
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -89,8 +89,10 @@ multidict==6.0.4
# via
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements/base.txt
networkx==3.1
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -86,7 +86,9 @@ multidict==6.0.4
# aiohttp
# yarl
networkx==3.1
# via -r requirements/base.txt
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -86,7 +86,9 @@ multidict==6.0.4
# aiohttp
# yarl
networkx==3.1
# via -r requirements/base.txt
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -94,7 +94,9 @@ multidict==6.0.4
# aiohttp
# yarl
networkx==3.1
# via -r requirements/base.txt
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -83,8 +83,10 @@ multidict==6.0.4
# via
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements/base.txt
networkx==3.1
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -83,8 +83,10 @@ multidict==6.0.4
# via
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements/base.txt
networkx==3.1
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -83,8 +83,10 @@ multidict==6.0.4
# via
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements/base.txt
networkx==3.1
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -91,8 +91,10 @@ multidict==6.0.4
# via
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements/base.txt
networkx==3.1
# via
# -c requirements/constraints.txt
# -r requirements/base.txt
packaging==23.1
# via -r requirements/base.txt
portend==3.1.0

View file

@ -14,18 +14,21 @@ if sys.version_info >= (3, 9, 5):
else:
import salt.ext.ipaddress as ipaddress
# importlib_metadata before version 3.3.0 does not include the functionality we need.
try:
import importlib_metadata
if sys.version_info >= (3, 10):
import importlib.metadata as importlib_metadata
else:
# importlib_metadata before version 3.3.0 does not include the functionality we need.
try:
import importlib_metadata
importlib_metadata_version = [
int(part)
for part in importlib_metadata.version("importlib_metadata").split(".")
if part.isdigit()
]
if tuple(importlib_metadata_version) < (3, 3, 0):
importlib_metadata_version = [
int(part)
for part in importlib_metadata.version("importlib_metadata").split(".")
if part.isdigit()
]
if tuple(importlib_metadata_version) < (3, 3, 0):
# Use the vendored importlib_metadata
import salt.ext.importlib_metadata as importlib_metadata
except ImportError:
# Use the vendored importlib_metadata
import salt.ext.importlib_metadata as importlib_metadata
except ImportError:
# Use the vendored importlib_metadata
import salt.ext.importlib_metadata as importlib_metadata

View file

@ -435,6 +435,7 @@ def high(data, **kwargs):
st_.push_active()
chunks, errors = st_.state.compile_high_data(data)
if errors:
__context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR
return errors
file_refs = salt.client.ssh.state.lowstate_file_refs(
chunks,

View file

@ -337,11 +337,13 @@ def sls(root, mods, saltenv="base", test=None, exclude=None, **kwargs):
errors += ext_errors
errors += st_.state.verify_high(high_data)
if errors:
__context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR
return errors
high_data, req_in_errors = st_.state.requisite_in(high_data)
errors += req_in_errors
if errors:
__context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR
return errors
high_data = st_.state.apply_exclude(high_data)
@ -349,6 +351,7 @@ def sls(root, mods, saltenv="base", test=None, exclude=None, **kwargs):
# Compile and verify the raw chunks
chunks, errors = st_.state.compile_high_data(high_data)
if errors:
__context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR
return errors
file_refs = salt.client.ssh.state.lowstate_file_refs(
chunks,

View file

@ -1921,6 +1921,7 @@ def sls_id(id_, mods, test=None, queue=None, state_events=None, **kwargs):
return errors
chunks, errors = st_.state.compile_high_data(high_)
if errors:
__context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR
return errors
ret = {}
for chunk in chunks:
@ -2031,6 +2032,7 @@ def show_low_sls(mods, test=None, queue=None, **kwargs):
return errors
ret, errors = st_.state.compile_high_data(high_)
if errors:
__context__["retcode"] = salt.defaults.exitcodes.EX_STATE_COMPILER_ERROR
return errors
# Work around Windows multiprocessing bug, set __opts__['test'] back to
# value from before this function was run.

View file

@ -13,7 +13,7 @@ from collections.abc import Generator, Iterable, Sequence
from enum import Enum, auto
from typing import TYPE_CHECKING, Any
import networkx as nx # pylint: disable=3rd-party-module-not-gated
import networkx as nx
log = logging.getLogger(__name__)
@ -96,6 +96,8 @@ class DependencyGraph:
between the states.
"""
__slots__ = ("dag", "nodes_lookup_map", "sls_to_nodes")
def __init__(self) -> None:
self.dag = nx.MultiDiGraph()
# a mapping to node_id to be able to find nodes with
@ -245,7 +247,6 @@ class DependencyGraph:
self, low: LowChunk, req_type: RequisiteType, req_key: str, req_val: str
) -> bool:
found = False
prereq_tag = None
has_prereq_node = low.get("__prereq__", False)
if req_key == "sls":
# Allow requisite tracking of entire sls files

View file

@ -704,6 +704,7 @@ def dependency_information(include_salt_cloud=False):
("M2Crypto", "M2Crypto", "version"),
("msgpack", "msgpack", "version"),
("msgpack-pure", "msgpack_pure", "version"),
("networkx", "networkx", "__version__"),
("pycrypto", "Crypto", "__version__"),
("pycryptodome", "Cryptodome", "version_info"),
("cryptography", "cryptography", "__version__"),

View file

@ -960,68 +960,13 @@ def test_mod_aggregate(minion_opts):
]
# Ensure that the require requisite from the
# figlet state finds its way into this state
assert "require" in low_ret
# figlet state doesn't find its way into this state
assert "require" not in low_ret
# Ensure pkgs were aggregated
assert low_ret["pkgs"] == ["sl", "hello"]
def test_mod_aggregate_order(minion_opts):
"""
Test to ensure that the state_aggregate setting correctly aggregates package installations
while respecting the 'require' requisite to enforce execution order.
"""
# Setup the chunks based on the provided scenario
chunks = [
{
"state": "pkg",
"name": "first packages",
"__id__": "first packages",
"pkgs": ["drpm"],
"fun": "installed",
"order": 1,
"__env__": "base",
"__sls__": "base",
},
{
"state": "test",
"name": "requirement",
"__id__": "requirement",
"fun": "nop",
"order": 2,
"__env__": "base",
"__sls__": "base",
},
{
"state": "pkg",
"name": "second packages",
"__id__": "second packages",
"pkgs": ["gc"],
"fun": "installed",
"order": 3,
"require": [{"test": "requirement"}],
"provider": "yumpkg",
"__env__": "base",
"__sls__": "base",
},
]
# Setup the State object
with patch("salt.state.State._gather_pillar"):
state_obj = salt.state.State(minion_opts)
state_obj.load_modules(chunks[-1])
state_obj.opts["state_aggregate"] = True # Ensure state aggregation is enabled
# Process each chunk with _mod_aggregate to simulate state execution
state_obj.call_chunks(chunks)
first_state_low = chunks[0]
last_state_low = chunks[-1]
# Verify that the requisites got aggregated as well
assert first_state_low["require"] == last_state_low["require"]
def test_verify_onlyif_cmd_opts_exclude(minion_opts):
"""
Verify cmd.run state arguments are properly excluded from cmd.retcode

View file

@ -1673,9 +1673,9 @@ class VirtualEnv:
kwargs.setdefault("stdout", subprocess.PIPE)
kwargs.setdefault("stderr", subprocess.PIPE)
kwargs.setdefault("universal_newlines", True)
env = kwargs.pop("env", None)
if env:
env = self.environ.copy().update(env)
if kwenv := kwargs.pop("env", None):
env = self.environ.copy()
env.update(kwenv)
else:
env = self.environ
proc = subprocess.run(args, check=False, env=env, **kwargs)
@ -1780,8 +1780,7 @@ class SaltVirtualEnv(VirtualEnv):
self.install(RUNTIME_VARS.CODE_DIR)
def install(self, *args, **kwargs):
env = self.environ.copy()
env.update(kwargs.pop("env", None) or {})
env = kwargs.pop("env", None) or {}
env["USE_STATIC_REQUIREMENTS"] = "1"
kwargs["env"] = env
return super().install(*args, **kwargs)