saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-10 14:51:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Add context aware change handling for file.managed state module function (#63347)

Browse source 
* add context aware change handling for file.managed state module function

* updates docstrings, add ignore comments, and add util tests

* update util tests

* adding additional documentation and test cases

* missed some places where a kwarg needed to be passed through

* handling pyupgrade changes

* using new md changelog format

* add context aware change handling for file.managed state module function

* updates docstrings, add ignore comments, and add util tests

* update util tests

* adding additional documentation and test cases

* missed some places where a kwarg needed to be passed through

* handling pyupgrade changes

* fix existence checking after merge conflict fix

* remove duplicate test from merge conflict resolution

* apparently using the github ui to resolve merge conflicts was a bad idea

---------

Co-authored-by: Daniel Wozniak <dwozniak@vmware.com>
This commit is contained in:
Nicholas Hughes 2024-01-22 16:34:42 -05:00 • committed by GitHub
parent 5098cf9710
commit b5c096920e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 759 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
changelog/63328.added.md Normal file
View file

@ -0,0 +1 @@
Add context aware change handling for file state module

293
salt/modules/file.py
View file

@ -4700,6 +4700,9 @@ def get_managed(
signed_by_all=None,
keyring=None,
gnupghome=None,
ignore_ordering=False,
ignore_whitespace=False,
ignore_comment_characters=None,
**kwargs,
):
"""
@ -4799,6 +4802,39 @@ def get_managed(
.. versionadded:: 3007.0
ignore_ordering
If ``True``, changes in line order will be ignored **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
.. versionadded:: 3007.0
ignore_whitespace
If ``True``, changes in whitespace will be ignored **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
Implies ``ignore_ordering=True``
.. versionadded:: 3007.0
ignore_comment_characters
If set to a chacter string, the presence of changes *after* that string
will be ignored in changes found in the file **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
Implies ``ignore_ordering=True``
.. versionadded:: 3007.0
CLI Example:
.. code-block:: bash
@ -5657,6 +5693,9 @@ def check_managed_changes(
serange=None,
verify_ssl=True,
follow_symlinks=False,
ignore_ordering=False,
ignore_whitespace=False,
ignore_comment_characters=None,
**kwargs,
):
"""
@ -5678,6 +5717,39 @@ def check_managed_changes(
.. versionadded:: 3005
ignore_ordering
If ``True``, changes in line order will be ignored **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
.. versionadded:: 3007.0
ignore_whitespace
If ``True``, changes in whitespace will be ignored **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
Implies ``ignore_ordering=True``
.. versionadded:: 3007.0
ignore_comment_characters
If set to a chacter string, the presence of changes *after* that string
will be ignored in changes found in the file **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
Implies ``ignore_ordering=True``
.. versionadded:: 3007.0
CLI Example:
.. code-block:: bash
@ -5709,6 +5781,9 @@ def check_managed_changes(
defaults,
skip_verify,
verify_ssl=verify_ssl,
ignore_ordering=ignore_ordering,
ignore_whitespace=ignore_whitespace,
ignore_comment_characters=ignore_comment_characters,
**kwargs,
)
@ -5744,6 +5819,9 @@ def check_managed_changes(
setype=setype,
serange=serange,
follow_symlinks=follow_symlinks,
ignore_ordering=ignore_ordering,
ignore_whitespace=ignore_whitespace,
ignore_comment_characters=ignore_comment_characters,
)
__clean_tmp(sfn)
return changes
@ -5766,6 +5844,9 @@ def check_file_meta(
serange=None,
verify_ssl=True,
follow_symlinks=False,
ignore_ordering=False,
ignore_whitespace=False,
ignore_comment_characters=None,
):
"""
Check for the changes in the file metadata.
@ -5848,8 +5929,42 @@ def check_file_meta(
of the file to which the symlink points.
.. versionadded:: 3005
ignore_ordering
If ``True``, changes in line order will be ignored **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
.. versionadded:: 3007.0
ignore_whitespace
If ``True``, changes in whitespace will be ignored **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
Implies ``ignore_ordering=True``
.. versionadded:: 3007.0
ignore_comment_characters
If set to a chacter string, the presence of changes *after* that string
will be ignored in changes found in the file **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
Implies ``ignore_ordering=True``
.. versionadded:: 3007.0
"""
changes = {}
has_changes = False
if not source_sum:
source_sum = dict()
@ -5864,6 +5979,8 @@ def check_file_meta(
if not lstats:
changes["newfile"] = name
if any([ignore_ordering, ignore_whitespace, ignore_comment_characters]):
return True, changes
return changes
if "hsum" in source_sum:
@ -5877,9 +5994,22 @@ def check_file_meta(
)
if sfn:
try:
changes["diff"] = get_diff(
name, sfn, template=True, show_filenames=False
)
if any(
[ignore_ordering, ignore_whitespace, ignore_comment_characters]
):
has_changes, changes["diff"] = get_diff(
name,
sfn,
template=True,
show_filenames=False,
ignore_ordering=ignore_ordering,
ignore_whitespace=ignore_whitespace,
ignore_comment_characters=ignore_comment_characters,
)
else:
changes["diff"] = get_diff(
name, sfn, template=True, show_filenames=False
)
except CommandExecutionError as exc:
changes["diff"] = exc.strerror
else:
@ -5905,7 +6035,17 @@ def check_file_meta(
tmp_.write(salt.utils.stringutils.to_str(contents))
# Compare the static contents with the named file
try:
differences = get_diff(name, tmp, show_filenames=False)
if any([ignore_ordering, ignore_whitespace, ignore_comment_characters]):
has_changes, differences = get_diff(
name,
tmp,
show_filenames=False,
ignore_ordering=ignore_ordering,
ignore_whitespace=ignore_whitespace,
ignore_comment_characters=ignore_comment_characters,
)
else:
differences = get_diff(name, tmp, show_filenames=False)
except CommandExecutionError as exc:
log.error("Failed to diff files: %s", exc)
differences = exc.strerror
@ -5968,6 +6108,9 @@ def check_file_meta(
if serange and serange != current_serange:
changes["selinux"] = {"range": serange}
if any([ignore_ordering, ignore_whitespace, ignore_comment_characters]):
return has_changes, changes
return changes
@ -5980,6 +6123,9 @@ def get_diff(
template=False,
source_hash_file1=None,
source_hash_file2=None,
ignore_ordering=False,
ignore_whitespace=False,
ignore_comment_characters=None,
):
"""
Return unified diff of two files
@ -6031,6 +6177,39 @@ def get_diff(
.. versionadded:: 2018.3.0
ignore_ordering
If ``True``, changes in line order will be ignored **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
.. versionadded:: 3007.0
ignore_whitespace
If ``True``, changes in whitespace will be ignored **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
Implies ``ignore_ordering=True``
.. versionadded:: 3007.0
ignore_comment_characters
If set to a chacter string, the presence of changes *after* that string
will be ignored in changes found in the file **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
Implies ``ignore_ordering=True``
.. versionadded:: 3007.0
CLI Examples:
.. code-block:: bash
@ -6089,9 +6268,20 @@ def get_diff(
else:
if show_filenames:
args.extend(paths)
ret = __utils__["stringutils.get_diff"](*args)
return ret
return ""
if any([ignore_ordering, ignore_whitespace, ignore_comment_characters]):
ret = __utils__["stringutils.get_conditional_diff"](
*args,
ignore_ordering=ignore_ordering,
ignore_whitespace=ignore_whitespace,
ignore_comment_characters=ignore_comment_characters,
)
else:
ret = __utils__["stringutils.get_diff"](*args)
elif any([ignore_ordering, ignore_whitespace, ignore_comment_characters]):
ret = (False, "")
else:
ret = ""
return ret
def manage_file(
@ -6128,6 +6318,9 @@ def manage_file(
signed_by_all=None,
keyring=None,
gnupghome=None,
ignore_ordering=False,
ignore_whitespace=False,
ignore_comment_characters=None,
**kwargs,
):
"""
@ -6319,6 +6512,39 @@ def manage_file(
.. versionadded:: 3007.0
ignore_ordering
If ``True``, changes in line order will be ignored **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
.. versionadded:: 3007.0
ignore_whitespace
If ``True``, changes in whitespace will be ignored **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
Implies ``ignore_ordering=True``
.. versionadded:: 3007.0
ignore_comment_characters
If set to a chacter string, the presence of changes *after* that string
will be ignored in changes found in the file **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
Implies ``ignore_ordering=True``
.. versionadded:: 3007.0
CLI Example:
.. code-block:: bash
@ -6330,6 +6556,7 @@ def manage_file(
"""
name = os.path.expanduser(name)
has_changes = False
check_web_source_hash = bool(
source
and urllib.parse.urlparse(source).scheme != "salt"
@ -6428,7 +6655,19 @@ def manage_file(
ret["changes"]["diff"] = "<show_changes=False>"
else:
try:
file_diff = get_diff(real_name, sfn, show_filenames=False)
if any(
[ignore_ordering, ignore_whitespace, ignore_comment_characters]
):
has_changes, file_diff = get_diff(
real_name,
sfn,
show_filenames=False,
ignore_ordering=ignore_ordering,
ignore_whitespace=ignore_whitespace,
ignore_comment_characters=ignore_comment_characters,
)
else:
file_diff = get_diff(real_name, sfn, show_filenames=False)
if file_diff:
ret["changes"]["diff"] = file_diff
except CommandExecutionError as exc:
@ -6465,13 +6704,25 @@ def manage_file(
tmp_.write(salt.utils.stringutils.to_bytes(contents))
try:
differences = get_diff(
real_name,
tmp,
show_filenames=False,
show_changes=show_changes,
template=True,
)
if any([ignore_ordering, ignore_whitespace, ignore_comment_characters]):
has_changes, differences = get_diff(
real_name,
tmp,
show_filenames=False,
show_changes=show_changes,
template=True,
ignore_ordering=ignore_ordering,
ignore_whitespace=ignore_whitespace,
ignore_comment_characters=ignore_comment_characters,
)
else:
differences = get_diff(
real_name,
tmp,
show_filenames=False,
show_changes=show_changes,
template=True,
)
except CommandExecutionError as exc:
ret.setdefault("warnings", []).append(
@ -6576,6 +6827,11 @@ def manage_file(
if ret["changes"]:
ret["comment"] = f"File {salt.utils.data.decode(name)} updated"
if (
any([ignore_ordering, ignore_whitespace, ignore_comment_characters])
and not has_changes
):
ret["skip_req"] = True
elif not ret["changes"] and ret["result"]:
ret["comment"] = "File {} is in the correct state".format(
@ -6772,6 +7028,13 @@ def manage_file(
if sfn:
__clean_tmp(sfn)
if (
any([ignore_ordering, ignore_whitespace, ignore_comment_characters])
and ret["changes"]
and not has_changes
):
ret["skip_req"] = True
return ret

25
salt/state.py
View file

@ -2990,6 +2990,12 @@ class State:
if tag not in run_dict:
req_stats.add("unmet")
continue
# A state can include a "skip_req" key in the return dict
# with a True value to skip triggering onchanges, watch, or
# other requisites which would result in a only running on a
# change or running mod_watch
if run_dict[tag].get("skip_req"):
req_stats.add("skip_req")
if r_state.startswith("onfail"):
if run_dict[tag]["result"] is True:
req_stats.add("onfail") # At least one state is OK
@ -3040,6 +3046,10 @@ class State:
status = "unmet"
elif "fail" in fun_stats:
status = "fail"
elif "skip_req" in fun_stats and (fun_stats & {"onchangesmet", "premet"}):
status = "skip_req"
elif "skip_req" in fun_stats and "change" in fun_stats:
status = "skip_watch"
elif "pre" in fun_stats:
if "premet" in fun_stats:
status = "met"
@ -3272,6 +3282,21 @@ class State:
self.pre[tag] = self.call(low, chunks, running)
else:
running[tag] = self.call(low, chunks, running)
elif status == "skip_req":
running[tag] = {
"changes": {},
"result": True,
"comment": "State was not run because requisites were skipped by another state",
"__run_num__": self.__run_num,
}
for key in ("__sls__", "__id__", "name"):
running[tag][key] = low.get(key)
elif status == "skip_watch" and not low.get("__prereq__"):
ret = self.call(low, chunks, running)
ret[
"comment"
] += " mod_watch was not run because requisites were skipped by another state"
running[tag] = ret
elif status == "fail":
# if the requisite that failed was due to a prereq on this low state
# show the normal error

65
salt/states/file.py
View file

@ -2321,6 +2321,9 @@ def managed(
signed_by_all=None,
keyring=None,
gnupghome=None,
ignore_ordering=False,
ignore_whitespace=False,
ignore_comment_characters=None,
**kwargs,
):
r"""
@ -2978,6 +2981,39 @@ def managed(
gnupghome
When verifying signatures, use this GnuPG home.
.. versionadded:: 3007.0
ignore_ordering
If ``True``, changes in line order will be ignored **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
.. versionadded:: 3007.0
ignore_whitespace
If ``True``, changes in whitespace will be ignored **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
Implies ``ignore_ordering=True``
.. versionadded:: 3007.0
ignore_comment_characters
If set to a chacter string, the presence of changes *after* that string
will be ignored in changes found in the file **ONLY** for the
purposes of triggering watch/onchanges requisites. Changes will still
be made to the file to bring it into alignment with requested state, and
also reported during the state run. This behavior is useful for bringing
existing application deployments under Salt configuration management
without disrupting production applications with a service restart.
Implies ``ignore_ordering=True``
.. versionadded:: 3007.0
"""
if "env" in kwargs:
@ -3000,6 +3036,8 @@ def managed(
if selinux is not None and not salt.utils.platform.is_linux():
return _error(ret, "The 'selinux' option is only supported on Linux")
has_changes = False
if signature or source_hash_sig:
# Fail early in case the gpg module is not present
try:
@ -3275,7 +3313,7 @@ def managed(
try:
if __opts__["test"]:
if "file.check_managed_changes" in __salt__:
ret["changes"] = __salt__["file.check_managed_changes"](
check_changes = __salt__["file.check_managed_changes"](
name,
source,
source_hash,
@ -3302,8 +3340,15 @@ def managed(
signed_by_all=signed_by_all,
keyring=keyring,
gnupghome=gnupghome,
ignore_ordering=ignore_ordering,
ignore_whitespace=ignore_whitespace,
ignore_comment_characters=ignore_comment_characters,
**kwargs,
)
if any([ignore_ordering, ignore_whitespace, ignore_comment_characters]):
has_changes, ret["changes"] = check_changes
else:
ret["changes"] = check_changes
if salt.utils.platform.is_windows():
try:
@ -3338,6 +3383,13 @@ def managed(
ret["result"] = True
ret["comment"] = f"The file {name} is in the correct state"
if (
any([ignore_ordering, ignore_whitespace, ignore_comment_characters])
and ret["changes"]
and not has_changes
):
ret["skip_req"] = True
return ret
# If the source is a list then find which file exists
@ -3430,6 +3482,9 @@ def managed(
signed_by_all=signed_by_all,
keyring=keyring,
gnupghome=gnupghome,
ignore_ordering=ignore_ordering,
ignore_whitespace=ignore_whitespace,
ignore_comment_characters=ignore_comment_characters,
**kwargs,
)
except Exception as exc: # pylint: disable=broad-except
@ -3453,6 +3508,11 @@ def managed(
if ret["changes"]:
# Reset ret
ret = {"changes": {}, "comment": "", "name": name, "result": True}
if (
any([ignore_ordering, ignore_whitespace, ignore_comment_characters])
and not has_changes
):
ret["skip_req"] = True
check_cmd_opts = {}
if "shell" in __grains__:
@ -3514,6 +3574,9 @@ def managed(
signed_by_all=signed_by_all,
keyring=keyring,
gnupghome=gnupghome,
ignore_ordering=ignore_ordering,
ignore_whitespace=ignore_whitespace,
ignore_comment_characters=ignore_comment_characters,
**kwargs,
)
except Exception as exc: # pylint: disable=broad-except

132
salt/utils/stringutils.py
View file

@ -50,7 +50,7 @@ def to_bytes(s, encoding=None, errors="strict"):
# raised, otherwise we would have already returned (or raised some
# other exception).
raise exc # pylint: disable=raising-bad-type
raise TypeError("expected str, bytes, or bytearray not {}".format(type(s)))
raise TypeError(f"expected str, bytes, or bytearray not {type(s)}")
def to_str(s, encoding=None, errors="strict", normalize=False):
@ -88,7 +88,7 @@ def to_str(s, encoding=None, errors="strict", normalize=False):
# raised, otherwise we would have already returned (or raised some
# other exception).
raise exc # pylint: disable=raising-bad-type
raise TypeError("expected str, bytes, or bytearray not {}".format(type(s)))
raise TypeError(f"expected str, bytes, or bytearray not {type(s)}")
def to_unicode(s, encoding=None, errors="strict", normalize=False):
@ -112,7 +112,7 @@ def to_unicode(s, encoding=None, errors="strict", normalize=False):
return _normalize(s)
elif isinstance(s, (bytes, bytearray)):
return _normalize(to_str(s, encoding, errors))
raise TypeError("expected str, bytes, or bytearray not {}".format(type(s)))
raise TypeError(f"expected str, bytes, or bytearray not {type(s)}")
@jinja_filter("str_to_num")
@ -301,7 +301,7 @@ def build_whitespace_split_regex(text):
for line in text.splitlines():
parts = [re.escape(s) for s in __build_parts(line)]
regex += r"(?:[\s]+)?{}(?:[\s]+)?".format(r"(?:[\s]+)?".join(parts))
return r"(?m)^{}$".format(regex)
return rf"(?m)^{regex}$"
def expr_match(line, expr):
@ -323,7 +323,7 @@ def expr_match(line, expr):
if fnmatch.fnmatch(line, expr):
return True
try:
if re.match(r"\A{}\Z".format(expr), line):
if re.match(rf"\A{expr}\Z", line):
return True
except re.error:
pass
@ -460,7 +460,7 @@ def print_cli(msg, retries=10, step=0.01):
except UnicodeEncodeError:
print(msg.encode("utf-8"))
except OSError as exc:
err = "{}".format(exc)
err = f"{exc}"
if exc.errno != errno.EPIPE:
if (
"temporarily unavailable" in err or exc.errno in (errno.EAGAIN,)
@ -508,26 +508,128 @@ def get_context(template, line, num_lines=5, marker=None):
return "---\n{}\n---".format("\n".join(buf))
def get_diff(a, b, *args, **kwargs):
def get_diff_list(a, b, *args, **kwargs):
"""
Perform diff on two iterables containing lines from two files, and return
the diff as as string. Lines are normalized to str types to avoid issues
the diff as a list. Lines are normalized to str types to avoid issues
with unicode on PY2.
"""
encoding = ("utf-8", "latin-1", __salt_system_encoding__)
# Late import to avoid circular import
import salt.utils.data
return "".join(
difflib.unified_diff(
salt.utils.data.decode_list(a, encoding=encoding),
salt.utils.data.decode_list(b, encoding=encoding),
*args,
**kwargs
)
return difflib.unified_diff(
salt.utils.data.decode_list(a, encoding=encoding),
salt.utils.data.decode_list(b, encoding=encoding),
*args,
**kwargs,
)
def get_diff(a, b, *args, **kwargs):
"""
Perform diff on two iterables containing lines from two files, and return
the diff as a string. Lines are normalized to str types to avoid issues
with unicode on PY2.
"""
return "".join(get_diff_list(a, b, *args, **kwargs))
def get_conditional_diff(
a,
b,
*args,
ignore_ordering=True,
ignore_whitespace=True,
ignore_comment_characters="#",
**kwargs,
):
"""
Perform diff on two iterables containing lines from two files, and return
the diff as as string. Lines are normalized to str types to avoid issues
with unicode on PY2.
Perform a diff on two iterables containing lines from two files, and return
the diff as a string. The resulting diff list will be filtered based on the
`ignore_ordering`, `ignore_whitespace`, and `ignore_comment_characters`
parameters. If any of those parameters are set, the function will check for
differences between the added and removed lines, after processing the diff
list.
If there are any differences, the function will return the boolean result
of the filtered diff list using the provided parameters as well as the
original diff list as a string. If there aren't any differences, the
function will return ``False`` and an empty string.
Parameters:
a: iterable
The first iterable to perform the diff against.
b: iterable
The second iterable to perform the diff against.
*args :
Additional arguments to pass to the ``get_diff_list`` function.
ignore_ordering (bool):
If True, the function will ignore the order of lines when checking for
differences.
ignore_whitespace (bool):
If True, the function will ignore leading and trailing white spaces when
checking for differences. Implies ``ignore_ordering``
ignore_comment_characters (str or list of str):
A string or list of strings representing comment characters. If
provided, the function will ignore any characters on the line after any
of these characters when checking for differences. Implies
``ignore_ordering``
**kwargs :
Additional keyword arguments to pass to the ``get_diff_list`` function.
Returns:
bool: The boolean result of the filtered diff list using the provided
parameters.
str: The diff of the two iterables as a string. Empty string if no
differences are found.
"""
if ignore_comment_characters is None:
ignore_comment_characters = []
elif isinstance(ignore_comment_characters, str):
ignore_comment_characters = [ignore_comment_characters]
elif not isinstance(ignore_comment_characters, list):
log.warning("ignore_comment_characters must be set to a string or list")
ignore_comment_characters = []
diff = list(get_diff_list(a, b, *args, **kwargs))
has_changes = False
if any([ignore_whitespace, ignore_ordering, ignore_comment_characters]):
adds = []
subs = []
for line in diff:
if line.startswith("+++") or line.startswith("---"):
continue
if line.startswith("+") or line.startswith("-"):
oper, *line = line
line = "".join(line)
for char in ignore_comment_characters:
if char in line:
# find 1st index of comment and delete everything after
line = line[: line.index(char)]
if ignore_whitespace:
line = line.strip()
if line and oper == "+":
adds.append(line)
elif line and oper == "-":
subs.append(line)
if sorted(adds) != sorted(subs):
has_changes = True
else:
has_changes = bool(diff)
return has_changes, "".join(diff)
@jinja_filter("to_snake_case")
def camel_to_snake_case(camel_input):
"""

60
tests/pytests/integration/states/test_file.py
View file

@ -1175,6 +1175,66 @@ def test_issue_62611(
assert state_run["result"] is True
def test_state_skip_req(
salt_master,
salt_call_cli,
tmp_path,
salt_minion,
):
target_path = tmp_path / "skip-req-file-target.txt"
target_path.write_text(
textwrap.dedent(
"""
foo=bar
# some comment
fizz=buzz
"""
)
)
name = "test_skip_req/skip_req"
sls_contents = """
modify_contents_with_ignore_params_to_skip:
file.managed:
- name: {}
- ignore_ordering: True
- ignore_whitespace: True
- ignore_comment_characters: '#'
- contents: |
fizz=buzz
foo=bar
this_req_should_not_trigger:
cmd.run:
- name: echo NEVER
- onchanges:
- file: modify_contents_with_ignore_params_to_skip
""".format(
target_path
)
sls_tempfile = salt_master.state_tree.base.temp_file(f"{name}.sls", sls_contents)
with sls_tempfile:
ret = salt_call_cli.run("state.apply", name.replace("/", "."))
assert ret.returncode == 0
assert ret.data
state_runs = list(ret.data.values())
# file.managed returns changes but doesn't trigger reqs
assert state_runs[0]["name"] == str(target_path)
assert state_runs[0]["result"] is True
assert state_runs[0]["changes"]
assert state_runs[0]["skip_req"] is True
# cmd.run is not run
assert state_runs[1]["name"] == "echo NEVER"
assert state_runs[1]["result"] is True
assert not state_runs[1]["changes"]
assert (
state_runs[1]["comment"]
== "State was not run because requisites were skipped by another state"
)
def test_contents_file(salt_master, salt_call_cli, tmp_path):
"""
test calling file.managed multiple times

215
tests/pytests/unit/utils/test_stringutils.py
View file

@ -9,7 +9,7 @@ import textwrap
import pytest
import salt.utils.stringutils
from tests.support.mock import patch
from tests.support.mock import MagicMock, patch
from tests.support.unit import LOREM_IPSUM
@ -770,3 +770,216 @@ def test_human_to_bytes_edge_cases():
assert salt.utils.stringutils.human_to_bytes("4 Kbytes") == 0
assert salt.utils.stringutils.human_to_bytes("9ib") == 0
assert salt.utils.stringutils.human_to_bytes("2HB") == 0
def test_get_conditional_diff_no_diff():
has_changes, diff = salt.utils.stringutils.get_conditional_diff(
"",
"",
ignore_ordering=True,
ignore_whitespace=True,
ignore_comment_characters="#",
)
assert has_changes is False
assert diff == ""
@pytest.mark.parametrize(
"ignore_ordering,ignore_whitespace,ignore_comment_characters,expected_changes",
(
(True, True, "#", False),
(True, True, None, True),
(True, False, "#", True),
(True, False, None, True),
(False, True, "#", False),
(False, True, None, True),
(False, False, "#", True),
(False, False, None, True),
),
)
def test_get_conditional_diff(
ignore_ordering, ignore_whitespace, ignore_comment_characters, expected_changes
):
mock_diff = textwrap.dedent(
"""
diff --git a/sample.txt b/sample.txt
index bf5a820..bea0a36 100644
--- a/sample.txt
+++ b/sample.txt
@@ -1,8 +1,5 @@
[section]
+stuff=things
things=stuff
-stuff=things
-
-foo=bar # comment about foo
-
-# fizzy comment
+foo=bar
fizz=buzz
"""
)
mock_diff_list = mock_diff.splitlines(True)
mock_get_diff_list = MagicMock(return_value=mock_diff_list)
with patch("salt.utils.stringutils.get_diff_list", mock_get_diff_list):
has_changes, diff = salt.utils.stringutils.get_conditional_diff(
"",
"",
ignore_ordering=ignore_ordering,
ignore_whitespace=ignore_whitespace,
ignore_comment_characters=ignore_comment_characters,
)
assert has_changes is expected_changes
assert diff == mock_diff
@pytest.mark.parametrize(
"ignore_ordering,ignore_whitespace,ignore_comment_characters,expected_changes",
(
(True, True, "#", False),
(True, True, None, False),
(True, False, "#", False),
(True, False, None, False),
(False, True, "#", False),
(False, True, None, False),
(False, False, "#", False),
(False, False, None, True),
),
)
def test_get_conditional_diff_ordering(
ignore_ordering, ignore_whitespace, ignore_comment_characters, expected_changes
):
mock_diff = textwrap.dedent(
"""
diff --git a/sample.txt b/sample.txt
index bf5a820..bc36b01 100644
--- a/sample.txt
+++ b/sample.txt
@@ -1,8 +1,8 @@
[section]
-things=stuff
stuff=things
-
-foo=bar # comment about foo
+things=stuff
# fizzy comment
fizz=buzz
+
+foo=bar # comment about foo
"""
)
mock_diff_list = mock_diff.splitlines(True)
mock_get_diff_list = MagicMock(return_value=mock_diff_list)
with patch("salt.utils.stringutils.get_diff_list", mock_get_diff_list):
has_changes, diff = salt.utils.stringutils.get_conditional_diff(
"",
"",
ignore_ordering=ignore_ordering,
ignore_whitespace=ignore_whitespace,
ignore_comment_characters=ignore_comment_characters,
)
assert has_changes is expected_changes
assert diff == mock_diff
@pytest.mark.parametrize(
"ignore_ordering,ignore_whitespace,ignore_comment_characters,expected_changes",
(
(True, True, "#", False),
(True, True, None, False),
(True, False, "#", True),
(True, False, None, True),
(False, True, "#", False),
(False, True, None, False),
(False, False, "#", True),
(False, False, None, True),
),
)
def test_get_conditional_diff_whitespace(
ignore_ordering, ignore_whitespace, ignore_comment_characters, expected_changes
):
mock_diff = textwrap.dedent(
"""
diff --git a/sample.txt b/sample.txt
index bf5a820..d17c48e 100644
--- a/sample.txt
+++ b/sample.txt
@@ -1,8 +1,7 @@
[section]
things=stuff
-stuff=things
+ stuff=things
foo=bar # comment about foo
-
# fizzy comment
fizz=buzz
"""
)
mock_diff_list = mock_diff.splitlines(True)
mock_get_diff_list = MagicMock(return_value=mock_diff_list)
with patch("salt.utils.stringutils.get_diff_list", mock_get_diff_list):
has_changes, diff = salt.utils.stringutils.get_conditional_diff(
"",
"",
ignore_ordering=ignore_ordering,
ignore_whitespace=ignore_whitespace,
ignore_comment_characters=ignore_comment_characters,
)
assert has_changes is expected_changes
assert diff == mock_diff
@pytest.mark.parametrize(
"ignore_ordering,ignore_whitespace,ignore_comment_characters,expected_changes",
(
(True, True, "#", False),
(True, True, None, True),
(True, False, "#", True),
(True, False, None, True),
(False, True, "#", False),
(False, True, None, True),
(False, False, "#", True),
(False, False, None, True),
),
)
def test_get_conditional_diff_comment(
ignore_ordering, ignore_whitespace, ignore_comment_characters, expected_changes
):
mock_diff = textwrap.dedent(
"""
diff --git a/sample.txt b/sample.txt
index bf5a820..fb1136a 100644
--- a/sample.txt
+++ b/sample.txt
@@ -1,8 +1,8 @@
[section]
-things=stuff
+things=stuff # comment about things
+# stuff comment
stuff=things
-foo=bar # comment about foo
+foo=bar
-# fizzy comment
fizz=buzz
"""
)
mock_diff_list = mock_diff.splitlines(True)
mock_get_diff_list = MagicMock(return_value=mock_diff_list)
with patch("salt.utils.stringutils.get_diff_list", mock_get_diff_list):
has_changes, diff = salt.utils.stringutils.get_conditional_diff(
"",
"",
ignore_ordering=ignore_ordering,
ignore_whitespace=ignore_whitespace,
ignore_comment_characters=ignore_comment_characters,
)
assert has_changes is expected_changes
assert diff == mock_diff