diff --git a/changelog/63328.added.md b/changelog/63328.added.md new file mode 100644 index 00000000000..76f2a919d6f --- /dev/null +++ b/changelog/63328.added.md @@ -0,0 +1 @@ +Add context aware change handling for file state module diff --git a/salt/modules/file.py b/salt/modules/file.py index 3a9f0669cbe..5c5cbc773d7 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -4700,6 +4700,9 @@ def get_managed( signed_by_all=None, keyring=None, gnupghome=None, + ignore_ordering=False, + ignore_whitespace=False, + ignore_comment_characters=None, **kwargs, ): """ @@ -4799,6 +4802,39 @@ def get_managed( .. versionadded:: 3007.0 + ignore_ordering + If ``True``, changes in line order will be ignored **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + + .. versionadded:: 3007.0 + + ignore_whitespace + If ``True``, changes in whitespace will be ignored **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + Implies ``ignore_ordering=True`` + + .. versionadded:: 3007.0 + + ignore_comment_characters + If set to a chacter string, the presence of changes *after* that string + will be ignored in changes found in the file **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + Implies ``ignore_ordering=True`` + + .. versionadded:: 3007.0 + CLI Example: .. code-block:: bash @@ -5657,6 +5693,9 @@ def check_managed_changes( serange=None, verify_ssl=True, follow_symlinks=False, + ignore_ordering=False, + ignore_whitespace=False, + ignore_comment_characters=None, **kwargs, ): """ @@ -5678,6 +5717,39 @@ def check_managed_changes( .. versionadded:: 3005 + ignore_ordering + If ``True``, changes in line order will be ignored **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + + .. versionadded:: 3007.0 + + ignore_whitespace + If ``True``, changes in whitespace will be ignored **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + Implies ``ignore_ordering=True`` + + .. versionadded:: 3007.0 + + ignore_comment_characters + If set to a chacter string, the presence of changes *after* that string + will be ignored in changes found in the file **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + Implies ``ignore_ordering=True`` + + .. versionadded:: 3007.0 + CLI Example: .. code-block:: bash @@ -5709,6 +5781,9 @@ def check_managed_changes( defaults, skip_verify, verify_ssl=verify_ssl, + ignore_ordering=ignore_ordering, + ignore_whitespace=ignore_whitespace, + ignore_comment_characters=ignore_comment_characters, **kwargs, ) @@ -5744,6 +5819,9 @@ def check_managed_changes( setype=setype, serange=serange, follow_symlinks=follow_symlinks, + ignore_ordering=ignore_ordering, + ignore_whitespace=ignore_whitespace, + ignore_comment_characters=ignore_comment_characters, ) __clean_tmp(sfn) return changes @@ -5766,6 +5844,9 @@ def check_file_meta( serange=None, verify_ssl=True, follow_symlinks=False, + ignore_ordering=False, + ignore_whitespace=False, + ignore_comment_characters=None, ): """ Check for the changes in the file metadata. @@ -5848,8 +5929,42 @@ def check_file_meta( of the file to which the symlink points. .. versionadded:: 3005 + + ignore_ordering + If ``True``, changes in line order will be ignored **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + + .. versionadded:: 3007.0 + + ignore_whitespace + If ``True``, changes in whitespace will be ignored **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + Implies ``ignore_ordering=True`` + + .. versionadded:: 3007.0 + + ignore_comment_characters + If set to a chacter string, the presence of changes *after* that string + will be ignored in changes found in the file **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + Implies ``ignore_ordering=True`` + + .. versionadded:: 3007.0 """ changes = {} + has_changes = False if not source_sum: source_sum = dict() @@ -5864,6 +5979,8 @@ def check_file_meta( if not lstats: changes["newfile"] = name + if any([ignore_ordering, ignore_whitespace, ignore_comment_characters]): + return True, changes return changes if "hsum" in source_sum: @@ -5877,9 +5994,22 @@ def check_file_meta( ) if sfn: try: - changes["diff"] = get_diff( - name, sfn, template=True, show_filenames=False - ) + if any( + [ignore_ordering, ignore_whitespace, ignore_comment_characters] + ): + has_changes, changes["diff"] = get_diff( + name, + sfn, + template=True, + show_filenames=False, + ignore_ordering=ignore_ordering, + ignore_whitespace=ignore_whitespace, + ignore_comment_characters=ignore_comment_characters, + ) + else: + changes["diff"] = get_diff( + name, sfn, template=True, show_filenames=False + ) except CommandExecutionError as exc: changes["diff"] = exc.strerror else: @@ -5905,7 +6035,17 @@ def check_file_meta( tmp_.write(salt.utils.stringutils.to_str(contents)) # Compare the static contents with the named file try: - differences = get_diff(name, tmp, show_filenames=False) + if any([ignore_ordering, ignore_whitespace, ignore_comment_characters]): + has_changes, differences = get_diff( + name, + tmp, + show_filenames=False, + ignore_ordering=ignore_ordering, + ignore_whitespace=ignore_whitespace, + ignore_comment_characters=ignore_comment_characters, + ) + else: + differences = get_diff(name, tmp, show_filenames=False) except CommandExecutionError as exc: log.error("Failed to diff files: %s", exc) differences = exc.strerror @@ -5968,6 +6108,9 @@ def check_file_meta( if serange and serange != current_serange: changes["selinux"] = {"range": serange} + if any([ignore_ordering, ignore_whitespace, ignore_comment_characters]): + return has_changes, changes + return changes @@ -5980,6 +6123,9 @@ def get_diff( template=False, source_hash_file1=None, source_hash_file2=None, + ignore_ordering=False, + ignore_whitespace=False, + ignore_comment_characters=None, ): """ Return unified diff of two files @@ -6031,6 +6177,39 @@ def get_diff( .. versionadded:: 2018.3.0 + ignore_ordering + If ``True``, changes in line order will be ignored **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + + .. versionadded:: 3007.0 + + ignore_whitespace + If ``True``, changes in whitespace will be ignored **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + Implies ``ignore_ordering=True`` + + .. versionadded:: 3007.0 + + ignore_comment_characters + If set to a chacter string, the presence of changes *after* that string + will be ignored in changes found in the file **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + Implies ``ignore_ordering=True`` + + .. versionadded:: 3007.0 + CLI Examples: .. code-block:: bash @@ -6089,9 +6268,20 @@ def get_diff( else: if show_filenames: args.extend(paths) - ret = __utils__["stringutils.get_diff"](*args) - return ret - return "" + if any([ignore_ordering, ignore_whitespace, ignore_comment_characters]): + ret = __utils__["stringutils.get_conditional_diff"]( + *args, + ignore_ordering=ignore_ordering, + ignore_whitespace=ignore_whitespace, + ignore_comment_characters=ignore_comment_characters, + ) + else: + ret = __utils__["stringutils.get_diff"](*args) + elif any([ignore_ordering, ignore_whitespace, ignore_comment_characters]): + ret = (False, "") + else: + ret = "" + return ret def manage_file( @@ -6128,6 +6318,9 @@ def manage_file( signed_by_all=None, keyring=None, gnupghome=None, + ignore_ordering=False, + ignore_whitespace=False, + ignore_comment_characters=None, **kwargs, ): """ @@ -6319,6 +6512,39 @@ def manage_file( .. versionadded:: 3007.0 + ignore_ordering + If ``True``, changes in line order will be ignored **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + + .. versionadded:: 3007.0 + + ignore_whitespace + If ``True``, changes in whitespace will be ignored **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + Implies ``ignore_ordering=True`` + + .. versionadded:: 3007.0 + + ignore_comment_characters + If set to a chacter string, the presence of changes *after* that string + will be ignored in changes found in the file **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + Implies ``ignore_ordering=True`` + + .. versionadded:: 3007.0 + CLI Example: .. code-block:: bash @@ -6330,6 +6556,7 @@ def manage_file( """ name = os.path.expanduser(name) + has_changes = False check_web_source_hash = bool( source and urllib.parse.urlparse(source).scheme != "salt" @@ -6428,7 +6655,19 @@ def manage_file( ret["changes"]["diff"] = "" else: try: - file_diff = get_diff(real_name, sfn, show_filenames=False) + if any( + [ignore_ordering, ignore_whitespace, ignore_comment_characters] + ): + has_changes, file_diff = get_diff( + real_name, + sfn, + show_filenames=False, + ignore_ordering=ignore_ordering, + ignore_whitespace=ignore_whitespace, + ignore_comment_characters=ignore_comment_characters, + ) + else: + file_diff = get_diff(real_name, sfn, show_filenames=False) if file_diff: ret["changes"]["diff"] = file_diff except CommandExecutionError as exc: @@ -6465,13 +6704,25 @@ def manage_file( tmp_.write(salt.utils.stringutils.to_bytes(contents)) try: - differences = get_diff( - real_name, - tmp, - show_filenames=False, - show_changes=show_changes, - template=True, - ) + if any([ignore_ordering, ignore_whitespace, ignore_comment_characters]): + has_changes, differences = get_diff( + real_name, + tmp, + show_filenames=False, + show_changes=show_changes, + template=True, + ignore_ordering=ignore_ordering, + ignore_whitespace=ignore_whitespace, + ignore_comment_characters=ignore_comment_characters, + ) + else: + differences = get_diff( + real_name, + tmp, + show_filenames=False, + show_changes=show_changes, + template=True, + ) except CommandExecutionError as exc: ret.setdefault("warnings", []).append( @@ -6576,6 +6827,11 @@ def manage_file( if ret["changes"]: ret["comment"] = f"File {salt.utils.data.decode(name)} updated" + if ( + any([ignore_ordering, ignore_whitespace, ignore_comment_characters]) + and not has_changes + ): + ret["skip_req"] = True elif not ret["changes"] and ret["result"]: ret["comment"] = "File {} is in the correct state".format( @@ -6772,6 +7028,13 @@ def manage_file( if sfn: __clean_tmp(sfn) + if ( + any([ignore_ordering, ignore_whitespace, ignore_comment_characters]) + and ret["changes"] + and not has_changes + ): + ret["skip_req"] = True + return ret diff --git a/salt/state.py b/salt/state.py index 45caa26b8d9..95fc5c524b7 100644 --- a/salt/state.py +++ b/salt/state.py @@ -2990,6 +2990,12 @@ class State: if tag not in run_dict: req_stats.add("unmet") continue + # A state can include a "skip_req" key in the return dict + # with a True value to skip triggering onchanges, watch, or + # other requisites which would result in a only running on a + # change or running mod_watch + if run_dict[tag].get("skip_req"): + req_stats.add("skip_req") if r_state.startswith("onfail"): if run_dict[tag]["result"] is True: req_stats.add("onfail") # At least one state is OK @@ -3040,6 +3046,10 @@ class State: status = "unmet" elif "fail" in fun_stats: status = "fail" + elif "skip_req" in fun_stats and (fun_stats & {"onchangesmet", "premet"}): + status = "skip_req" + elif "skip_req" in fun_stats and "change" in fun_stats: + status = "skip_watch" elif "pre" in fun_stats: if "premet" in fun_stats: status = "met" @@ -3272,6 +3282,21 @@ class State: self.pre[tag] = self.call(low, chunks, running) else: running[tag] = self.call(low, chunks, running) + elif status == "skip_req": + running[tag] = { + "changes": {}, + "result": True, + "comment": "State was not run because requisites were skipped by another state", + "__run_num__": self.__run_num, + } + for key in ("__sls__", "__id__", "name"): + running[tag][key] = low.get(key) + elif status == "skip_watch" and not low.get("__prereq__"): + ret = self.call(low, chunks, running) + ret[ + "comment" + ] += " mod_watch was not run because requisites were skipped by another state" + running[tag] = ret elif status == "fail": # if the requisite that failed was due to a prereq on this low state # show the normal error diff --git a/salt/states/file.py b/salt/states/file.py index 8daf99c5f08..d9b68038eda 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -2321,6 +2321,9 @@ def managed( signed_by_all=None, keyring=None, gnupghome=None, + ignore_ordering=False, + ignore_whitespace=False, + ignore_comment_characters=None, **kwargs, ): r""" @@ -2978,6 +2981,39 @@ def managed( gnupghome When verifying signatures, use this GnuPG home. + .. versionadded:: 3007.0 + + ignore_ordering + If ``True``, changes in line order will be ignored **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + + .. versionadded:: 3007.0 + + ignore_whitespace + If ``True``, changes in whitespace will be ignored **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + Implies ``ignore_ordering=True`` + + .. versionadded:: 3007.0 + + ignore_comment_characters + If set to a chacter string, the presence of changes *after* that string + will be ignored in changes found in the file **ONLY** for the + purposes of triggering watch/onchanges requisites. Changes will still + be made to the file to bring it into alignment with requested state, and + also reported during the state run. This behavior is useful for bringing + existing application deployments under Salt configuration management + without disrupting production applications with a service restart. + Implies ``ignore_ordering=True`` + .. versionadded:: 3007.0 """ if "env" in kwargs: @@ -3000,6 +3036,8 @@ def managed( if selinux is not None and not salt.utils.platform.is_linux(): return _error(ret, "The 'selinux' option is only supported on Linux") + has_changes = False + if signature or source_hash_sig: # Fail early in case the gpg module is not present try: @@ -3275,7 +3313,7 @@ def managed( try: if __opts__["test"]: if "file.check_managed_changes" in __salt__: - ret["changes"] = __salt__["file.check_managed_changes"]( + check_changes = __salt__["file.check_managed_changes"]( name, source, source_hash, @@ -3302,8 +3340,15 @@ def managed( signed_by_all=signed_by_all, keyring=keyring, gnupghome=gnupghome, + ignore_ordering=ignore_ordering, + ignore_whitespace=ignore_whitespace, + ignore_comment_characters=ignore_comment_characters, **kwargs, ) + if any([ignore_ordering, ignore_whitespace, ignore_comment_characters]): + has_changes, ret["changes"] = check_changes + else: + ret["changes"] = check_changes if salt.utils.platform.is_windows(): try: @@ -3338,6 +3383,13 @@ def managed( ret["result"] = True ret["comment"] = f"The file {name} is in the correct state" + if ( + any([ignore_ordering, ignore_whitespace, ignore_comment_characters]) + and ret["changes"] + and not has_changes + ): + ret["skip_req"] = True + return ret # If the source is a list then find which file exists @@ -3430,6 +3482,9 @@ def managed( signed_by_all=signed_by_all, keyring=keyring, gnupghome=gnupghome, + ignore_ordering=ignore_ordering, + ignore_whitespace=ignore_whitespace, + ignore_comment_characters=ignore_comment_characters, **kwargs, ) except Exception as exc: # pylint: disable=broad-except @@ -3453,6 +3508,11 @@ def managed( if ret["changes"]: # Reset ret ret = {"changes": {}, "comment": "", "name": name, "result": True} + if ( + any([ignore_ordering, ignore_whitespace, ignore_comment_characters]) + and not has_changes + ): + ret["skip_req"] = True check_cmd_opts = {} if "shell" in __grains__: @@ -3514,6 +3574,9 @@ def managed( signed_by_all=signed_by_all, keyring=keyring, gnupghome=gnupghome, + ignore_ordering=ignore_ordering, + ignore_whitespace=ignore_whitespace, + ignore_comment_characters=ignore_comment_characters, **kwargs, ) except Exception as exc: # pylint: disable=broad-except diff --git a/salt/utils/stringutils.py b/salt/utils/stringutils.py index 30ca46fee5c..3f8ee1e2de8 100644 --- a/salt/utils/stringutils.py +++ b/salt/utils/stringutils.py @@ -50,7 +50,7 @@ def to_bytes(s, encoding=None, errors="strict"): # raised, otherwise we would have already returned (or raised some # other exception). raise exc # pylint: disable=raising-bad-type - raise TypeError("expected str, bytes, or bytearray not {}".format(type(s))) + raise TypeError(f"expected str, bytes, or bytearray not {type(s)}") def to_str(s, encoding=None, errors="strict", normalize=False): @@ -88,7 +88,7 @@ def to_str(s, encoding=None, errors="strict", normalize=False): # raised, otherwise we would have already returned (or raised some # other exception). raise exc # pylint: disable=raising-bad-type - raise TypeError("expected str, bytes, or bytearray not {}".format(type(s))) + raise TypeError(f"expected str, bytes, or bytearray not {type(s)}") def to_unicode(s, encoding=None, errors="strict", normalize=False): @@ -112,7 +112,7 @@ def to_unicode(s, encoding=None, errors="strict", normalize=False): return _normalize(s) elif isinstance(s, (bytes, bytearray)): return _normalize(to_str(s, encoding, errors)) - raise TypeError("expected str, bytes, or bytearray not {}".format(type(s))) + raise TypeError(f"expected str, bytes, or bytearray not {type(s)}") @jinja_filter("str_to_num") @@ -301,7 +301,7 @@ def build_whitespace_split_regex(text): for line in text.splitlines(): parts = [re.escape(s) for s in __build_parts(line)] regex += r"(?:[\s]+)?{}(?:[\s]+)?".format(r"(?:[\s]+)?".join(parts)) - return r"(?m)^{}$".format(regex) + return rf"(?m)^{regex}$" def expr_match(line, expr): @@ -323,7 +323,7 @@ def expr_match(line, expr): if fnmatch.fnmatch(line, expr): return True try: - if re.match(r"\A{}\Z".format(expr), line): + if re.match(rf"\A{expr}\Z", line): return True except re.error: pass @@ -460,7 +460,7 @@ def print_cli(msg, retries=10, step=0.01): except UnicodeEncodeError: print(msg.encode("utf-8")) except OSError as exc: - err = "{}".format(exc) + err = f"{exc}" if exc.errno != errno.EPIPE: if ( "temporarily unavailable" in err or exc.errno in (errno.EAGAIN,) @@ -508,26 +508,128 @@ def get_context(template, line, num_lines=5, marker=None): return "---\n{}\n---".format("\n".join(buf)) -def get_diff(a, b, *args, **kwargs): +def get_diff_list(a, b, *args, **kwargs): """ Perform diff on two iterables containing lines from two files, and return - the diff as as string. Lines are normalized to str types to avoid issues + the diff as a list. Lines are normalized to str types to avoid issues with unicode on PY2. """ encoding = ("utf-8", "latin-1", __salt_system_encoding__) # Late import to avoid circular import import salt.utils.data - return "".join( - difflib.unified_diff( - salt.utils.data.decode_list(a, encoding=encoding), - salt.utils.data.decode_list(b, encoding=encoding), - *args, - **kwargs - ) + return difflib.unified_diff( + salt.utils.data.decode_list(a, encoding=encoding), + salt.utils.data.decode_list(b, encoding=encoding), + *args, + **kwargs, ) +def get_diff(a, b, *args, **kwargs): + """ + Perform diff on two iterables containing lines from two files, and return + the diff as a string. Lines are normalized to str types to avoid issues + with unicode on PY2. + """ + return "".join(get_diff_list(a, b, *args, **kwargs)) + + +def get_conditional_diff( + a, + b, + *args, + ignore_ordering=True, + ignore_whitespace=True, + ignore_comment_characters="#", + **kwargs, +): + """ + Perform diff on two iterables containing lines from two files, and return + the diff as as string. Lines are normalized to str types to avoid issues + with unicode on PY2. + + Perform a diff on two iterables containing lines from two files, and return + the diff as a string. The resulting diff list will be filtered based on the + `ignore_ordering`, `ignore_whitespace`, and `ignore_comment_characters` + parameters. If any of those parameters are set, the function will check for + differences between the added and removed lines, after processing the diff + list. + + If there are any differences, the function will return the boolean result + of the filtered diff list using the provided parameters as well as the + original diff list as a string. If there aren't any differences, the + function will return ``False`` and an empty string. + + Parameters: + a: iterable + The first iterable to perform the diff against. + b: iterable + The second iterable to perform the diff against. + *args : + Additional arguments to pass to the ``get_diff_list`` function. + ignore_ordering (bool): + If True, the function will ignore the order of lines when checking for + differences. + ignore_whitespace (bool): + If True, the function will ignore leading and trailing white spaces when + checking for differences. Implies ``ignore_ordering`` + ignore_comment_characters (str or list of str): + A string or list of strings representing comment characters. If + provided, the function will ignore any characters on the line after any + of these characters when checking for differences. Implies + ``ignore_ordering`` + **kwargs : + Additional keyword arguments to pass to the ``get_diff_list`` function. + + Returns: + bool: The boolean result of the filtered diff list using the provided + parameters. + str: The diff of the two iterables as a string. Empty string if no + differences are found. + """ + if ignore_comment_characters is None: + ignore_comment_characters = [] + elif isinstance(ignore_comment_characters, str): + ignore_comment_characters = [ignore_comment_characters] + elif not isinstance(ignore_comment_characters, list): + log.warning("ignore_comment_characters must be set to a string or list") + ignore_comment_characters = [] + + diff = list(get_diff_list(a, b, *args, **kwargs)) + + has_changes = False + if any([ignore_whitespace, ignore_ordering, ignore_comment_characters]): + adds = [] + subs = [] + for line in diff: + if line.startswith("+++") or line.startswith("---"): + continue + if line.startswith("+") or line.startswith("-"): + oper, *line = line + line = "".join(line) + + for char in ignore_comment_characters: + if char in line: + # find 1st index of comment and delete everything after + line = line[: line.index(char)] + + if ignore_whitespace: + line = line.strip() + + if line and oper == "+": + adds.append(line) + elif line and oper == "-": + subs.append(line) + + if sorted(adds) != sorted(subs): + has_changes = True + else: + has_changes = bool(diff) + + return has_changes, "".join(diff) + + @jinja_filter("to_snake_case") def camel_to_snake_case(camel_input): """ diff --git a/tests/pytests/integration/states/test_file.py b/tests/pytests/integration/states/test_file.py index 4b286a854f4..3e907b69e73 100644 --- a/tests/pytests/integration/states/test_file.py +++ b/tests/pytests/integration/states/test_file.py @@ -1175,6 +1175,66 @@ def test_issue_62611( assert state_run["result"] is True +def test_state_skip_req( + salt_master, + salt_call_cli, + tmp_path, + salt_minion, +): + target_path = tmp_path / "skip-req-file-target.txt" + target_path.write_text( + textwrap.dedent( + """ + foo=bar + # some comment + fizz=buzz + """ + ) + ) + name = "test_skip_req/skip_req" + + sls_contents = """ + modify_contents_with_ignore_params_to_skip: + file.managed: + - name: {} + - ignore_ordering: True + - ignore_whitespace: True + - ignore_comment_characters: '#' + - contents: | + fizz=buzz + foo=bar + + this_req_should_not_trigger: + cmd.run: + - name: echo NEVER + - onchanges: + - file: modify_contents_with_ignore_params_to_skip + """.format( + target_path + ) + + sls_tempfile = salt_master.state_tree.base.temp_file(f"{name}.sls", sls_contents) + + with sls_tempfile: + ret = salt_call_cli.run("state.apply", name.replace("/", ".")) + assert ret.returncode == 0 + assert ret.data + state_runs = list(ret.data.values()) + # file.managed returns changes but doesn't trigger reqs + assert state_runs[0]["name"] == str(target_path) + assert state_runs[0]["result"] is True + assert state_runs[0]["changes"] + assert state_runs[0]["skip_req"] is True + # cmd.run is not run + assert state_runs[1]["name"] == "echo NEVER" + assert state_runs[1]["result"] is True + assert not state_runs[1]["changes"] + assert ( + state_runs[1]["comment"] + == "State was not run because requisites were skipped by another state" + ) + + def test_contents_file(salt_master, salt_call_cli, tmp_path): """ test calling file.managed multiple times diff --git a/tests/pytests/unit/utils/test_stringutils.py b/tests/pytests/unit/utils/test_stringutils.py index 5b26d3473b1..66a24a0a1b4 100644 --- a/tests/pytests/unit/utils/test_stringutils.py +++ b/tests/pytests/unit/utils/test_stringutils.py @@ -9,7 +9,7 @@ import textwrap import pytest import salt.utils.stringutils -from tests.support.mock import patch +from tests.support.mock import MagicMock, patch from tests.support.unit import LOREM_IPSUM @@ -770,3 +770,216 @@ def test_human_to_bytes_edge_cases(): assert salt.utils.stringutils.human_to_bytes("4 Kbytes") == 0 assert salt.utils.stringutils.human_to_bytes("9ib") == 0 assert salt.utils.stringutils.human_to_bytes("2HB") == 0 + + +def test_get_conditional_diff_no_diff(): + has_changes, diff = salt.utils.stringutils.get_conditional_diff( + "", + "", + ignore_ordering=True, + ignore_whitespace=True, + ignore_comment_characters="#", + ) + assert has_changes is False + assert diff == "" + + +@pytest.mark.parametrize( + "ignore_ordering,ignore_whitespace,ignore_comment_characters,expected_changes", + ( + (True, True, "#", False), + (True, True, None, True), + (True, False, "#", True), + (True, False, None, True), + (False, True, "#", False), + (False, True, None, True), + (False, False, "#", True), + (False, False, None, True), + ), +) +def test_get_conditional_diff( + ignore_ordering, ignore_whitespace, ignore_comment_characters, expected_changes +): + mock_diff = textwrap.dedent( + """ + diff --git a/sample.txt b/sample.txt + index bf5a820..bea0a36 100644 + --- a/sample.txt + +++ b/sample.txt + @@ -1,8 +1,5 @@ + [section] + +stuff=things + things=stuff + -stuff=things + - + -foo=bar # comment about foo + - + -# fizzy comment + +foo=bar + fizz=buzz + """ + ) + mock_diff_list = mock_diff.splitlines(True) + mock_get_diff_list = MagicMock(return_value=mock_diff_list) + + with patch("salt.utils.stringutils.get_diff_list", mock_get_diff_list): + has_changes, diff = salt.utils.stringutils.get_conditional_diff( + "", + "", + ignore_ordering=ignore_ordering, + ignore_whitespace=ignore_whitespace, + ignore_comment_characters=ignore_comment_characters, + ) + assert has_changes is expected_changes + assert diff == mock_diff + + +@pytest.mark.parametrize( + "ignore_ordering,ignore_whitespace,ignore_comment_characters,expected_changes", + ( + (True, True, "#", False), + (True, True, None, False), + (True, False, "#", False), + (True, False, None, False), + (False, True, "#", False), + (False, True, None, False), + (False, False, "#", False), + (False, False, None, True), + ), +) +def test_get_conditional_diff_ordering( + ignore_ordering, ignore_whitespace, ignore_comment_characters, expected_changes +): + mock_diff = textwrap.dedent( + """ + diff --git a/sample.txt b/sample.txt + index bf5a820..bc36b01 100644 + --- a/sample.txt + +++ b/sample.txt + @@ -1,8 +1,8 @@ + [section] + -things=stuff + stuff=things + - + -foo=bar # comment about foo + +things=stuff + + # fizzy comment + fizz=buzz + + + +foo=bar # comment about foo + """ + ) + mock_diff_list = mock_diff.splitlines(True) + mock_get_diff_list = MagicMock(return_value=mock_diff_list) + + with patch("salt.utils.stringutils.get_diff_list", mock_get_diff_list): + has_changes, diff = salt.utils.stringutils.get_conditional_diff( + "", + "", + ignore_ordering=ignore_ordering, + ignore_whitespace=ignore_whitespace, + ignore_comment_characters=ignore_comment_characters, + ) + assert has_changes is expected_changes + assert diff == mock_diff + + +@pytest.mark.parametrize( + "ignore_ordering,ignore_whitespace,ignore_comment_characters,expected_changes", + ( + (True, True, "#", False), + (True, True, None, False), + (True, False, "#", True), + (True, False, None, True), + (False, True, "#", False), + (False, True, None, False), + (False, False, "#", True), + (False, False, None, True), + ), +) +def test_get_conditional_diff_whitespace( + ignore_ordering, ignore_whitespace, ignore_comment_characters, expected_changes +): + mock_diff = textwrap.dedent( + """ + diff --git a/sample.txt b/sample.txt + index bf5a820..d17c48e 100644 + --- a/sample.txt + +++ b/sample.txt + @@ -1,8 +1,7 @@ + [section] + things=stuff + -stuff=things + + stuff=things + + foo=bar # comment about foo + - + # fizzy comment + fizz=buzz + """ + ) + mock_diff_list = mock_diff.splitlines(True) + mock_get_diff_list = MagicMock(return_value=mock_diff_list) + + with patch("salt.utils.stringutils.get_diff_list", mock_get_diff_list): + has_changes, diff = salt.utils.stringutils.get_conditional_diff( + "", + "", + ignore_ordering=ignore_ordering, + ignore_whitespace=ignore_whitespace, + ignore_comment_characters=ignore_comment_characters, + ) + assert has_changes is expected_changes + assert diff == mock_diff + + +@pytest.mark.parametrize( + "ignore_ordering,ignore_whitespace,ignore_comment_characters,expected_changes", + ( + (True, True, "#", False), + (True, True, None, True), + (True, False, "#", True), + (True, False, None, True), + (False, True, "#", False), + (False, True, None, True), + (False, False, "#", True), + (False, False, None, True), + ), +) +def test_get_conditional_diff_comment( + ignore_ordering, ignore_whitespace, ignore_comment_characters, expected_changes +): + mock_diff = textwrap.dedent( + """ + diff --git a/sample.txt b/sample.txt + index bf5a820..fb1136a 100644 + --- a/sample.txt + +++ b/sample.txt + @@ -1,8 +1,8 @@ + [section] + -things=stuff + +things=stuff # comment about things + +# stuff comment + stuff=things + + -foo=bar # comment about foo + +foo=bar + + -# fizzy comment + fizz=buzz + """ + ) + mock_diff_list = mock_diff.splitlines(True) + mock_get_diff_list = MagicMock(return_value=mock_diff_list) + + with patch("salt.utils.stringutils.get_diff_list", mock_get_diff_list): + has_changes, diff = salt.utils.stringutils.get_conditional_diff( + "", + "", + ignore_ordering=ignore_ordering, + ignore_whitespace=ignore_whitespace, + ignore_comment_characters=ignore_comment_characters, + ) + assert has_changes is expected_changes + assert diff == mock_diff