saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge branch '3006.x' into merge-forward

Browse source
This commit is contained in:
Daniel A. Wozniak 2023-12-22 22:12:49 -07:00
commit 874698b9fb
13 changed files with 109 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
changelog/65691.fixed.md Normal file
View file

@ -0,0 +1 @@
Fix boto execution module loading

1
changelog/65692.fixed.md Normal file
View file

@ -0,0 +1 @@
Removed PR 65185 changes since incomplete solution

7
salt/fileclient.py
View file

@ -42,15 +42,12 @@ log = logging.getLogger(__name__)
MAX_FILENAME_LENGTH = 255
def get_file_client(opts, pillar=False, force_local=False):
def get_file_client(opts, pillar=False):
"""
Read in the ``file_client`` option and return the correct type of file
server
"""
if force_local:
client = "local"
else:
client = opts.get("file_client", "remote")
client = opts.get("file_client", "remote")
if pillar and client == "local":
client = "pillar"

4
salt/loader/__init__.py
View file

@ -324,6 +324,10 @@ def minion_mods(
pack_self="__salt__",
)
# Allow the usage of salt dunder in utils modules.
if utils and isinstance(utils, LazyLoader):
utils.pack["__salt__"] = ret
# Load any provider overrides from the configuration file providers option
# Note: Providers can be pkg, service, user or group - not to be confused
# with cloud providers.

9
salt/loader/context.py
View file

@ -12,6 +12,8 @@ except ImportError:
# Py<3.7
import contextvars
import salt.exceptions
DEFAULT_CTX_VAR = "loader_ctxvar"
loader_ctxvar = contextvars.ContextVar(DEFAULT_CTX_VAR)
@ -69,7 +71,12 @@ class NamedLoaderContext(collections.abc.MutableMapping):
return loader.pack[self.name]
if self.name == loader.pack_self:
return loader
return loader.pack[self.name]
try:
return loader.pack[self.name]
except KeyError:
raise salt.exceptions.LoaderError(
f"LazyLoader does not have a packed value for: {self.name}"
)
def get(self, key, default=None):
return self.value().get(key, default)

42
salt/minion.py
View file

@ -115,29 +115,6 @@ log = logging.getLogger(__name__)
# 6. Handle publications
def _sync_grains(opts):
# need sync of custom grains as may be used in pillar compilation
# if coming up initially and remote client, the first sync _grains
# doesn't have opts["master_uri"] set yet during the sync, so need
# to force local, otherwise will throw an exception when attempting
# to retrieve opts["master_uri"] when retrieving key for remote communication
# in addition opts sometimes does not contain extmod_whitelist and extmod_blacklist
# hence set those to defaults, empty dict, if not part of opts, as ref'd in
# salt.utils.extmod sync function
if opts.get("extmod_whitelist", None) is None:
opts["extmod_whitelist"] = {}
if opts.get("extmod_blacklist", None) is None:
opts["extmod_blacklist"] = {}
if opts.get("file_client", "remote") == "remote" and not opts.get(
"master_uri", None
):
salt.utils.extmods.sync(opts, "grains", force_local=True)
else:
salt.utils.extmods.sync(opts, "grains")
def resolve_dns(opts, fallback=True):
"""
Resolves the master_ip and master_uri options
@ -945,7 +922,6 @@ class SMinion(MinionBase):
# Late setup of the opts grains, so we can log from the grains module
import salt.loader
_sync_grains(opts)
opts["grains"] = salt.loader.grains(opts)
super().__init__(opts)
@ -1287,6 +1263,7 @@ class Minion(MinionBase):
self.ready = False
self.jid_queue = [] if jid_queue is None else jid_queue
self.periodic_callbacks = {}
self.req_channel = None
if io_loop is None:
self.io_loop = tornado.ioloop.IOLoop.current()
@ -1397,6 +1374,16 @@ class Minion(MinionBase):
"""
Return a future which will complete when you are connected to a master
"""
if hasattr(self, "pub_channel") and self.pub_channel:
self.pub_channel.on_recv(None)
if hasattr(self.pub_channel, "auth"):
self.pub_channel.auth.invalidate()
if hasattr(self.pub_channel, "close"):
self.pub_channel.close()
if hasattr(self, "req_channel") and self.req_channel:
self.req_channel.close()
self.req_channel = None
# Consider refactoring so that eval_master does not have a subtle side-effect on the contents of the opts array
master, self.pub_channel = yield self.eval_master(
self.opts, self.timeout, self.safe, failed
@ -2872,7 +2859,9 @@ class Minion(MinionBase):
self.pub_channel.auth.invalidate()
if hasattr(self.pub_channel, "close"):
self.pub_channel.close()
del self.pub_channel
if hasattr(self, "req_channel") and self.req_channel:
self.req_channel.close()
self.req_channel = None
# if eval_master finds a new master for us, self.connected
# will be True again on successful master authentication
@ -3304,6 +3293,9 @@ class Minion(MinionBase):
self.pub_channel.on_recv(None)
self.pub_channel.close()
del self.pub_channel
if hasattr(self, "req_channel") and self.req_channel:
self.req_channel.close()
self.req_channel = None
if hasattr(self, "periodic_callbacks"):
for cb in self.periodic_callbacks.values():
cb.stop()

5
salt/utils/extmods.py
View file

@ -39,7 +39,6 @@ def sync(
saltenv=None,
extmod_whitelist=None,
extmod_blacklist=None,
force_local=False,
):
"""
Sync custom modules into the extension_modules directory
@ -83,9 +82,7 @@ def sync(
"Cannot create cache module directory %s. Check permissions.",
mod_dir,
)
with salt.fileclient.get_file_client(
opts, pillar=False, force_local=force_local
) as fileclient:
with salt.fileclient.get_file_client(opts) as fileclient:
for sub_env in saltenv:
log.info("Syncing %s for environment '%s'", form, sub_env)
cache = []

4
tests/pytests/integration/cli/test_matcher.py
View file

@ -502,7 +502,9 @@ def test_salt_documentation(salt_cli, salt_minion):
"""
Test to see if we're supporting --doc
"""
ret = salt_cli.run("-d", "test", minion_tgt=salt_minion.id)
# Setting an explicity long timeout otherwise this test may fail when the
# system is under load.
ret = salt_cli.run("-d", "test", minion_tgt=salt_minion.id, _timeout=90)
assert ret.returncode == 0
assert "test.ping" in ret.data

71
tests/pytests/integration/cli/test_salt_call.py
View file

@ -429,74 +429,3 @@ def test_local_salt_call_no_function_no_retcode(salt_call_cli):
assert "test" in ret.data
assert ret.data["test"] == "'test' is not available."
assert "test.echo" in ret.data
def test_state_highstate_custom_grains(salt_master, salt_minion_factory):
"""
This test ensure that custom grains in salt://_grains are loaded before pillar compilation
to ensure that any use of custom grains in pillar files are available, this implies that
a sync of grains occurs before loading the regular /etc/salt/grains or configuration file
grains, as well as the usual grains.
Note: cannot use salt_minion and salt_call_cli, since these will be loaded before
the pillar and custom_grains files are written, hence using salt_minion_factory.
"""
pillar_top_sls = """
base:
'*':
- defaults
"""
pillar_defaults_sls = """
mypillar: "{{ grains['custom_grain'] }}"
"""
salt_top_sls = """
base:
'*':
- test
"""
salt_test_sls = """
"donothing":
test.nop: []
"""
salt_custom_grains_py = """
def main():
return {'custom_grain': 'test_value'}
"""
assert salt_master.is_running()
with salt_minion_factory.started():
salt_minion = salt_minion_factory
salt_call_cli = salt_minion_factory.salt_call_cli()
with salt_minion.pillar_tree.base.temp_file(
"top.sls", pillar_top_sls
), salt_minion.pillar_tree.base.temp_file(
"defaults.sls", pillar_defaults_sls
), salt_minion.state_tree.base.temp_file(
"top.sls", salt_top_sls
), salt_minion.state_tree.base.temp_file(
"test.sls", salt_test_sls
), salt_minion.state_tree.base.temp_file(
"_grains/custom_grain.py", salt_custom_grains_py
):
ret = salt_call_cli.run("--local", "state.highstate")
assert ret.returncode == 0
ret = salt_call_cli.run("--local", "pillar.items")
assert ret.returncode == 0
assert ret.data
pillar_items = ret.data
assert "mypillar" in pillar_items
assert pillar_items["mypillar"] == "test_value"
def test_salt_call_versions(salt_call_cli, caplog):
"""
Call test.versions without '--local' to test grains
are sync'd without any missing keys in opts
"""
with caplog.at_level(logging.DEBUG):
ret = salt_call_cli.run("test.versions")
assert ret.returncode == 0
assert "Failed to sync grains module: 'master_uri'" not in caplog.messages

18
tests/pytests/integration/minion/test_return_retries.py
View file

@ -3,6 +3,8 @@ import time
import pytest
from saltfactories.utils import random_string
import salt.utils.files
@pytest.fixture(scope="function")
def salt_minion_retry(salt_master, salt_minion_id):
@ -53,13 +55,15 @@ def test_publish_retry(salt_master, salt_minion_retry, salt_cli, salt_run_cli):
@pytest.mark.slow_test
def test_pillar_timeout(salt_master_factory):
cmd = """
python -c "import time; time.sleep(3.0); print('{\\"foo\\": \\"bar\\"}');\"
""".strip()
def test_pillar_timeout(salt_master_factory, tmp_path):
cmd = 'print(\'{"foo": "bar"}\');\n'
with salt.utils.files.fopen(tmp_path / "script.py", "w") as fp:
fp.write(cmd)
master_overrides = {
"ext_pillar": [
{"cmd_json": cmd},
{"cmd_json": f"python {tmp_path / 'script.py'}"},
],
"auto_accept": True,
"worker_threads": 2,
@ -103,8 +107,10 @@ def test_pillar_timeout(salt_master_factory):
cli = master.salt_cli()
sls_tempfile = master.state_tree.base.temp_file(f"{sls_name}.sls", sls_contents)
with master.started(), minion1.started(), minion2.started(), minion3.started(), minion4.started(), sls_tempfile:
cmd = 'import time; time.sleep(6); print(\'{"foo": "bang"}\');\n'
with salt.utils.files.fopen(tmp_path / "script.py", "w") as fp:
fp.write(cmd)
proc = cli.run("state.sls", sls_name, minion_tgt="*")
print(proc)
# At least one minion should have a Pillar timeout
assert proc.returncode == 1
minion_timed_out = False

3
tests/pytests/scenarios/multimaster/beacons/test_inotify.py
View file

@ -46,6 +46,7 @@ def setup_beacons(mm_master_1_salt_cli, salt_mm_minion_1, inotify_test_path):
"inotify",
beacon_data=[{"files": {str(inotify_test_path): {"mask": ["create"]}}}],
minion_tgt=salt_mm_minion_1.id,
timeout=60,
)
assert ret.returncode == 0
log.debug("Inotify beacon add returned: %s", ret.data or ret.stdout)
@ -95,7 +96,7 @@ def test_beacons_duplicate_53344(
# Since beacons will be executed both together, we wait for the status beacon event
# which means that, the inotify becacon was executed too
start_time = setup_beacons
expected_tag = "salt/beacon/{}/status/*".format(salt_mm_minion_1.id)
expected_tag = f"salt/beacon/{salt_mm_minion_1.id}/status/*"
expected_patterns = [
(salt_mm_master_1.id, expected_tag),
(salt_mm_master_2.id, expected_tag),

20
tests/pytests/unit/loader/test_loader.py
View file

@ -10,6 +10,7 @@ import textwrap
import pytest
import salt.exceptions
import salt.loader
import salt.loader.lazy
@ -62,3 +63,22 @@ def test_raw_mod_functions():
ret = salt.loader.raw_mod(opts, "grains", "get")
for k, v in ret.items():
assert isinstance(v, salt.loader.lazy.LoadedFunc)
def test_named_loader_context_name_not_packed(tmp_path):
opts = {
"optimization_order": [0],
}
contents = """
from salt.loader.dunder import loader_context
__not_packed__ = loader_context.named_context("__not_packed__")
def foobar():
return __not_packed__["not.packed"]()
"""
with pytest.helpers.temp_file("mymod.py", contents, directory=tmp_path):
loader = salt.loader.LazyLoader([tmp_path], opts)
with pytest.raises(
salt.exceptions.LoaderError,
match="LazyLoader does not have a packed value for: __not_packed__",
):
loader["mymod.foobar"]()

38
tests/pytests/unit/loader/test_loading_modules.py Normal file
View file

@ -0,0 +1,38 @@
import pytest
import salt.loader
import salt.loader.lazy
import salt.modules.boto_vpc
import salt.modules.virt
@pytest.fixture
def minion_mods(minion_opts):
utils = salt.loader.utils(minion_opts)
return salt.loader.minion_mods(minion_opts, utils=utils)
@pytest.mark.skipif(
not salt.modules.boto_vpc.HAS_BOTO, reason="boto must be installed."
)
def test_load_boto_vpc(minion_mods):
func = None
try:
func = minion_mods["boto_vpc.check_vpc"]
except KeyError:
pytest.fail("loader should not raise KeyError")
assert func is not None
assert isinstance(func, salt.loader.lazy.LoadedFunc)
@pytest.mark.skipif(
not salt.modules.virt.HAS_LIBVIRT, reason="libvirt-python must be installed."
)
def test_load_virt(minion_mods):
func = None
try:
func = minion_mods["virt.ctrl_alt_del"]
except KeyError:
pytest.fail("loader should not raise KeyError")
assert func is not None
assert isinstance(func, salt.loader.lazy.LoadedFunc)