saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Fix salt-ssh closing client connection and add defaults functional test

Browse source
This commit is contained in:
Megan Wilhite 2023-03-31 14:05:50 -06:00
parent e4b1ca3e30
commit 62966ec126
6 changed files with 144 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

83
salt/client/ssh/__init__.py
View file

@ -1682,50 +1682,51 @@ def mod_data(fsclient):
"returners",
]
ret = {}
envs = fsclient.envs()
ver_base = ""
for env in envs:
files = fsclient.file_list(env)
for ref in sync_refs:
mods_data = {}
pref = "_{}".format(ref)
for fn_ in sorted(files):
if fn_.startswith(pref):
if fn_.endswith((".py", ".so", ".pyx")):
full = salt.utils.url.create(fn_)
mod_path = fsclient.cache_file(full, env)
if not os.path.isfile(mod_path):
continue
mods_data[os.path.basename(fn_)] = mod_path
chunk = salt.utils.hashutils.get_hash(mod_path)
ver_base += chunk
if mods_data:
if ref in ret:
ret[ref].update(mods_data)
else:
ret[ref] = mods_data
if not ret:
return {}
with fsclient:
envs = fsclient.envs()
ver_base = ""
for env in envs:
files = fsclient.file_list(env)
for ref in sync_refs:
mods_data = {}
pref = "_{}".format(ref)
for fn_ in sorted(files):
if fn_.startswith(pref):
if fn_.endswith((".py", ".so", ".pyx")):
full = salt.utils.url.create(fn_)
mod_path = fsclient.cache_file(full, env)
if not os.path.isfile(mod_path):
continue
mods_data[os.path.basename(fn_)] = mod_path
chunk = salt.utils.hashutils.get_hash(mod_path)
ver_base += chunk
if mods_data:
if ref in ret:
ret[ref].update(mods_data)
else:
ret[ref] = mods_data
if not ret:
return {}
ver_base = salt.utils.stringutils.to_bytes(ver_base)
ver_base = salt.utils.stringutils.to_bytes(ver_base)
ver = hashlib.sha1(ver_base).hexdigest()
ext_tar_path = os.path.join(
fsclient.opts["cachedir"], "ext_mods.{}.tgz".format(ver)
)
mods = {"version": ver, "file": ext_tar_path}
if os.path.isfile(ext_tar_path):
ver = hashlib.sha1(ver_base).hexdigest()
ext_tar_path = os.path.join(
fsclient.opts["cachedir"], "ext_mods.{}.tgz".format(ver)
)
mods = {"version": ver, "file": ext_tar_path}
if os.path.isfile(ext_tar_path):
return mods
tfp = tarfile.open(ext_tar_path, "w:gz")
verfile = os.path.join(fsclient.opts["cachedir"], "ext_mods.ver")
with salt.utils.files.fopen(verfile, "w+") as fp_:
fp_.write(ver)
tfp.add(verfile, "ext_version")
for ref in ret:
for fn_ in ret[ref]:
tfp.add(ret[ref][fn_], os.path.join(ref, fn_))
tfp.close()
return mods
tfp = tarfile.open(ext_tar_path, "w:gz")
verfile = os.path.join(fsclient.opts["cachedir"], "ext_mods.ver")
with salt.utils.files.fopen(verfile, "w+") as fp_:
fp_.write(ver)
tfp.add(verfile, "ext_version")
for ref in ret:
for fn_ in ret[ref]:
tfp.add(ret[ref][fn_], os.path.join(ref, fn_))
tfp.close()
return mods
def ssh_version():

55
salt/client/ssh/state.py
View file

@ -223,37 +223,36 @@ def prep_trans_tar(
env_root = os.path.join(gendir, saltenv)
if not os.path.isdir(env_root):
os.makedirs(env_root)
with file_client:
for ref in file_refs[saltenv]:
for name in ref:
short = salt.utils.url.parse(name)[0].lstrip("/")
cache_dest = os.path.join(cache_dest_root, short)
try:
path = file_client.cache_file(name, saltenv, cachedir=cachedir)
except OSError:
path = ""
if path:
tgt = os.path.join(env_root, short)
for ref in file_refs[saltenv]:
for name in ref:
short = salt.utils.url.parse(name)[0].lstrip("/")
cache_dest = os.path.join(cache_dest_root, short)
try:
path = file_client.cache_file(name, saltenv, cachedir=cachedir)
except OSError:
path = ""
if path:
tgt = os.path.join(env_root, short)
tgt_dir = os.path.dirname(tgt)
if not os.path.isdir(tgt_dir):
os.makedirs(tgt_dir)
shutil.copy(path, tgt)
continue
try:
files = file_client.cache_dir(name, saltenv, cachedir=cachedir)
except OSError:
files = ""
if files:
for filename in files:
fn = filename[
len(file_client.get_cachedir(cache_dest)) :
].strip("/")
tgt = os.path.join(env_root, short, fn)
tgt_dir = os.path.dirname(tgt)
if not os.path.isdir(tgt_dir):
os.makedirs(tgt_dir)
shutil.copy(path, tgt)
continue
try:
files = file_client.cache_dir(name, saltenv, cachedir=cachedir)
except OSError:
files = ""
if files:
for filename in files:
fn = filename[
len(file_client.get_cachedir(cache_dest)) :
].strip("/")
tgt = os.path.join(env_root, short, fn)
tgt_dir = os.path.dirname(tgt)
if not os.path.isdir(tgt_dir):
os.makedirs(tgt_dir)
shutil.copy(filename, tgt)
continue
shutil.copy(filename, tgt)
continue
try:
# cwd may not exist if it was removed but salt was run from it
cwd = os.getcwd()

58
salt/client/ssh/wrapper/saltcheck.py
View file

@ -28,7 +28,10 @@ def update_master_cache(states, saltenv="base"):
# Setup for copying states to gendir
gendir = tempfile.mkdtemp()
trans_tar = salt.utils.files.mkstemp()
client = salt.fileclient.get_file_client(__opts__)
if "cp.fileclient_{}".format(id(__opts__)) not in __context__:
__context__[
"cp.fileclient_{}".format(id(__opts__))
] = salt.fileclient.get_file_client(__opts__)
# generate cp.list_states output and save to gendir
cp_output = salt.utils.json.dumps(__salt__["cp.list_states"]())
@ -56,33 +59,34 @@ def update_master_cache(states, saltenv="base"):
log.debug("copying %s to %s", state_name, gendir)
qualified_name = salt.utils.url.create(state_name, saltenv)
# Duplicate cp.get_dir to gendir
with client:
copy_result = client.get_dir(qualified_name, gendir, saltenv)
if copy_result:
copy_result = [
dir.replace(gendir, state_cache) for dir in copy_result
]
copy_result_output = salt.utils.json.dumps(copy_result)
with salt.utils.files.fopen(file_copy_file, "w") as fp:
fp.write(copy_result_output)
already_processed.append(state_name)
copy_result = __context__["cp.fileclient_{}".format(id(__opts__))].get_dir(
qualified_name, gendir, saltenv
)
if copy_result:
copy_result = [dir.replace(gendir, state_cache) for dir in copy_result]
copy_result_output = salt.utils.json.dumps(copy_result)
with salt.utils.files.fopen(file_copy_file, "w") as fp:
fp.write(copy_result_output)
already_processed.append(state_name)
else:
# If files were not copied, assume state.file.sls was given and just copy state
state_name = os.path.dirname(state_name)
file_copy_file = os.path.join(gendir, state_name + ".copy")
if state_name in already_processed:
log.debug("Already cached state for %s", state_name)
else:
# If files were not copied, assume state.file.sls was given and just copy state
state_name = os.path.dirname(state_name)
file_copy_file = os.path.join(gendir, state_name + ".copy")
if state_name in already_processed:
log.debug("Already cached state for %s", state_name)
else:
qualified_name = salt.utils.url.create(state_name, saltenv)
copy_result = client.get_dir(qualified_name, gendir, saltenv)
if copy_result:
copy_result = [
dir.replace(gendir, state_cache) for dir in copy_result
]
copy_result_output = salt.utils.json.dumps(copy_result)
with salt.utils.files.fopen(file_copy_file, "w") as fp:
fp.write(copy_result_output)
already_processed.append(state_name)
qualified_name = salt.utils.url.create(state_name, saltenv)
copy_result = __context__[
"cp.fileclient_{}".format(id(__opts__))
].get_dir(qualified_name, gendir, saltenv)
if copy_result:
copy_result = [
dir.replace(gendir, state_cache) for dir in copy_result
]
copy_result_output = salt.utils.json.dumps(copy_result)
with salt.utils.files.fopen(file_copy_file, "w") as fp:
fp.write(copy_result_output)
already_processed.append(state_name)
# turn gendir into tarball and remove gendir
try:

23
salt/modules/chroot.py
View file

@ -242,13 +242,14 @@ def _create_and_execute_salt_state(root, chunks, file_refs, test, hash_type):
"""
# Create the tar containing the state pkg and relevant files.
salt.client.ssh.wrapper.state._cleanup_slsmod_low_data(chunks)
trans_tar = salt.client.ssh.state.prep_trans_tar(
salt.fileclient.get_file_client(__opts__),
chunks,
file_refs,
__pillar__.value(),
root,
)
with salt.fileclient.get_file_client(__opts__) as client:
trans_tar = salt.client.ssh.state.prep_trans_tar(
client,
chunks,
file_refs,
__pillar__.value(),
root,
)
trans_tar_sum = salt.utils.hashutils.get_hash(trans_tar, hash_type)
ret = None
@ -314,14 +315,14 @@ def sls(root, mods, saltenv="base", test=None, exclude=None, **kwargs):
# Clone the options data and apply some default values. May not be
# needed, as this module just delegate
opts = salt.utils.state.get_sls_opts(__opts__, **kwargs)
st_ = salt.client.ssh.state.SSHHighState(
opts, pillar, __salt__, salt.fileclient.get_file_client(__opts__)
)
if isinstance(mods, str):
mods = mods.split(",")
with st_:
with salt.client.ssh.state.SSHHighState(
opts, pillar, __salt__, salt.fileclient.get_file_client(__opts__)
) as st_:
high_data, errors = st_.render_highstate({saltenv: mods})
if exclude:
if isinstance(exclude, str):

6
salt/modules/dockermod.py
View file

@ -6648,7 +6648,8 @@ def _mk_fileclient():
"""
Create a file client and add it to the context.
"""
return salt.fileclient.get_file_client(__opts__)
if "cp.fileclient" not in __context__:
__context__["cp.fileclient"] = salt.fileclient.get_file_client(__opts__)
def _generate_tmp_path():
@ -6664,8 +6665,9 @@ def _prepare_trans_tar(name, sls_opts, mods=None, pillar=None, extra_filerefs=""
# reuse it from salt.ssh, however this function should
# be somewhere else
refs = salt.client.ssh.state.lowstate_file_refs(chunks, extra_filerefs)
_mk_fileclient()
trans_tar = salt.client.ssh.state.prep_trans_tar(
_mk_fileclient(), chunks, refs, pillar, name
__context__["cp.fileclient"], chunks, refs, pillar, name
)
return trans_tar

28
tests/pytests/functional/modules/state/test_state.py
View file

@ -1009,3 +1009,31 @@ def test_issue_62264_requisite_not_found(state, state_tree):
for state_return in ret:
assert state_return.result is True
assert "The following requisites were not found" not in state_return.comment
def test_state_sls_defaults(state, state_tree):
""" """
json_contents = """
{
"users": {
"root": 1
}
}
"""
sls_contents = """
{% set test = salt['defaults.get']('core:users:root') %}
echo {{ test }}:
cmd.run
"""
state_id = "cmd_|-echo 1_|-echo 1_|-run"
core_dir = state_tree / "core"
with pytest.helpers.temp_file(
core_dir / "defaults.json", json_contents, state_tree
):
with pytest.helpers.temp_file(core_dir / "test.sls", sls_contents, state_tree):
ret = state.sls("core.test")
assert state_id in ret
for state_return in ret:
assert state_return.result is True
assert "echo 1" in state_return.comment