diff --git a/changelog/56441.fixed.md b/changelog/56441.fixed.md new file mode 100644 index 00000000000..489ad80f770 --- /dev/null +++ b/changelog/56441.fixed.md @@ -0,0 +1 @@ +Fixed config.get does not support merge option with salt-ssh diff --git a/salt/client/ssh/wrapper/config.py b/salt/client/ssh/wrapper/config.py index dcc00ceb2c3..a6db176453c 100644 --- a/salt/client/ssh/wrapper/config.py +++ b/salt/client/ssh/wrapper/config.py @@ -2,17 +2,22 @@ Return config information """ +import copy +import fnmatch +import logging import os -import re +import urllib.parse import salt.syspaths as syspaths import salt.utils.data import salt.utils.files +import salt.utils.sdb as sdb + +log = logging.getLogger(__name__) # Set up the default values for all systems DEFAULTS = { "mongo.db": "salt", - "mongo.host": "salt", "mongo.password": "", "mongo.port": 27017, "mongo.user": "", @@ -38,9 +43,12 @@ DEFAULTS = { "solr.num_backups": 1, "poudriere.config": "/usr/local/etc/poudriere.conf", "poudriere.config_dir": "/usr/local/etc/poudriere.d", + "ldap.uri": "", "ldap.server": "localhost", "ldap.port": "389", "ldap.tls": False, + "ldap.no_verify": False, + "ldap.anonymous": True, "ldap.scope": 2, "ldap.attrs": None, "ldap.binddn": "", @@ -51,6 +59,11 @@ DEFAULTS = { "tunnel": False, "images": os.path.join(syspaths.SRV_ROOT_DIR, "salt-images"), }, + "docker.exec_driver": "docker-exec", + "docker.compare_container_networks": { + "static": ["Aliases", "Links", "IPAMConfig"], + "automatic": ["IPAddress", "Gateway", "GlobalIPv6Address", "IPv6Gateway"], + }, } @@ -96,15 +109,66 @@ def valid_fileproto(uri): salt '*' config.valid_fileproto salt://path/to/file """ - try: - return bool(re.match("^(?:salt|https?|ftp)://", uri)) - except Exception: # pylint: disable=broad-except - return False + return urllib.parse.urlparse(uri).scheme in salt.utils.files.VALID_PROTOS -def option(value, default="", omit_opts=False, omit_master=False, omit_pillar=False): +def option( + value, + default=None, + omit_opts=False, + omit_grains=False, + omit_pillar=False, + omit_master=False, + omit_all=False, + wildcard=False, +): """ - Pass in a generic option and receive the value that will be assigned + Returns the setting for the specified config value. The priority for + matches is the same as in :py:func:`config.get `, + only this function does not recurse into nested data structures. Another + difference between this function and :py:func:`config.get + ` is that it comes with a set of "sane defaults". + To view these, you can run the following command: + + .. code-block:: bash + + salt '*' config.option '*' omit_all=True wildcard=True + + default + The default value if no match is found. If not specified, then the + fallback default will be an empty string, unless ``wildcard=True``, in + which case the return will be an empty dictionary. + + omit_opts : False + Pass as ``True`` to exclude matches from the minion configuration file + + omit_grains : False + Pass as ``True`` to exclude matches from the grains + + omit_pillar : False + Pass as ``True`` to exclude matches from the pillar data + + omit_master : False + Pass as ``True`` to exclude matches from the master configuration file + + omit_all : True + Shorthand to omit all of the above and return matches only from the + "sane defaults". + + .. versionadded:: 3000 + + wildcard : False + If used, this will perform pattern matching on keys. Note that this + will also significantly change the return data. Instead of only a value + being returned, a dictionary mapping the matched keys to their values + is returned. For example, using ``wildcard=True`` with a ``key`` of + ``'foo.ba*`` could return a dictionary like so: + + .. code-block:: python + + {'foo.bar': True, 'foo.baz': False} + + .. versionadded:: 3000 CLI Example: @@ -112,18 +176,48 @@ def option(value, default="", omit_opts=False, omit_master=False, omit_pillar=Fa salt '*' config.option redis.host """ - if not omit_opts: - if value in __opts__: - return __opts__[value] - if not omit_master: - if value in __pillar__.get("master", {}): - return __pillar__["master"][value] - if not omit_pillar: - if value in __pillar__: - return __pillar__[value] - if value in DEFAULTS: - return DEFAULTS[value] - return default + if omit_all: + omit_opts = omit_grains = omit_pillar = omit_master = True + + if default is None: + default = "" if not wildcard else {} + + if not wildcard: + if not omit_opts: + if value in __opts__: + return __opts__[value] + if not omit_grains: + if value in __grains__: + return __grains__[value] + if not omit_pillar: + if value in __pillar__: + return __pillar__[value] + if not omit_master: + if value in __pillar__.get("master", {}): + return __pillar__["master"][value] + if value in DEFAULTS: + return DEFAULTS[value] + + # No match + return default + else: + # We need to do the checks in the reverse order so that minion opts + # takes precedence + ret = {} + for omit, data in ( + (omit_master, __pillar__.get("master", {})), + (omit_pillar, __pillar__), + (omit_grains, __grains__), + (omit_opts, __opts__), + ): + if not omit: + ret.update({x: data[x] for x in fnmatch.filter(data, value)}) + # Check the DEFAULTS as well to see if the pattern matches it + for item in (x for x in fnmatch.filter(DEFAULTS, value) if x not in ret): + ret[item] = DEFAULTS[item] + + # If no matches, return the default + return ret or default def merge(value, default="", omit_opts=False, omit_master=False, omit_pillar=False): @@ -171,54 +265,223 @@ def merge(value, default="", omit_opts=False, omit_master=False, omit_pillar=Fal ret = list(ret) + list(tmp) if ret is None and value in DEFAULTS: return DEFAULTS[value] - return ret or default + if ret is None: + return default + return ret -def get(key, default=""): +def get( + key, + default="", + delimiter=":", + merge=None, + omit_opts=False, + omit_pillar=False, + omit_master=False, + omit_grains=False, +): """ .. versionadded:: 0.14.0 - Attempt to retrieve the named value from opts, pillar, grains of the master - config, if the named value is not available return the passed default. - The default return is an empty string. + Attempt to retrieve the named value from the minion config file, pillar, + grains or the master config. If the named value is not available, return + the value specified by the ``default`` argument. If this argument is not + specified, ``default`` falls back to an empty string. - The value can also represent a value in a nested dict using a ":" delimiter - for the dict. This means that if a dict looks like this:: + Values can also be retrieved from nested dictionaries. Assume the below + data structure: + + .. code-block:: python {'pkg': {'apache': 'httpd'}} - To retrieve the value associated with the apache key in the pkg dict this - key can be passed:: + To retrieve the value associated with the ``apache`` key, in the + sub-dictionary corresponding to the ``pkg`` key, the following command can + be used: - pkg:apache + .. code-block:: bash - This routine traverses these data stores in this order: + salt myminion config.get pkg:apache - - Local minion config (opts) + The ``:`` (colon) is used to represent a nested dictionary level. + + .. versionchanged:: 2015.5.0 + The ``delimiter`` argument was added, to allow delimiters other than + ``:`` to be used. + + This function traverses these data stores in this order, returning the + first match found: + + - Minion configuration - Minion's grains - - Minion's pillar - - Master config + - Minion's pillar data + - Master configuration (requires :conf_minion:`pillar_opts` to be set to + ``True`` in Minion config file in order to work) + + This means that if there is a value that is going to be the same for the + majority of minions, it can be configured in the Master config file, and + then overridden using the grains, pillar, or Minion config file. + + Adding config options to the Master or Minion configuration file is easy: + + .. code-block:: yaml + + my-config-option: value + cafe-menu: + - egg and bacon + - egg sausage and bacon + - egg and spam + - egg bacon and spam + - egg bacon sausage and spam + - spam bacon sausage and spam + - spam egg spam spam bacon and spam + - spam sausage spam spam bacon spam tomato and spam + + .. note:: + Minion configuration options built into Salt (like those defined + :ref:`here `) will *always* be defined in + the Minion configuration and thus *cannot be overridden by grains or + pillar data*. However, additional (user-defined) configuration options + (as in the above example) will not be in the Minion configuration by + default and thus can be overridden using grains/pillar data by leaving + the option out of the minion config file. + + **Arguments** + + delimiter + .. versionadded:: 2015.5.0 + + Override the delimiter used to separate nested levels of a data + structure. + + merge + .. versionadded:: 2015.5.0 + + If passed, this parameter will change the behavior of the function so + that, instead of traversing each data store above in order and + returning the first match, the data stores are first merged together + and then searched. The pillar data is merged into the master config + data, then the grains are merged, followed by the Minion config data. + The resulting data structure is then searched for a match. This allows + for configurations to be more flexible. + + .. note:: + + The merging described above does not mean that grain data will end + up in the Minion's pillar data, or pillar data will end up in the + master config data, etc. The data is just combined for the purposes + of searching an amalgam of the different data stores. + + The supported merge strategies are as follows: + + - **recurse** - If a key exists in both dictionaries, and the new value + is not a dictionary, it is replaced. Otherwise, the sub-dictionaries + are merged together into a single dictionary, recursively on down, + following the same criteria. For example: + + .. code-block:: python + + >>> dict1 = {'foo': {'bar': 1, 'qux': True}, + 'hosts': ['a', 'b', 'c'], + 'only_x': None} + >>> dict2 = {'foo': {'baz': 2, 'qux': False}, + 'hosts': ['d', 'e', 'f'], + 'only_y': None} + >>> merged + {'foo': {'bar': 1, 'baz': 2, 'qux': False}, + 'hosts': ['d', 'e', 'f'], + 'only_dict1': None, + 'only_dict2': None} + + - **overwrite** - If a key exists in the top level of both + dictionaries, the new value completely overwrites the old. For + example: + + .. code-block:: python + + >>> dict1 = {'foo': {'bar': 1, 'qux': True}, + 'hosts': ['a', 'b', 'c'], + 'only_x': None} + >>> dict2 = {'foo': {'baz': 2, 'qux': False}, + 'hosts': ['d', 'e', 'f'], + 'only_y': None} + >>> merged + {'foo': {'baz': 2, 'qux': False}, + 'hosts': ['d', 'e', 'f'], + 'only_dict1': None, + 'only_dict2': None} CLI Example: .. code-block:: bash salt '*' config.get pkg:apache + salt '*' config.get lxc.container_profile:centos merge=recurse """ - ret = salt.utils.data.traverse_dict_and_list(__opts__, key, "_|-") - if ret != "_|-": - return ret - ret = salt.utils.data.traverse_dict_and_list(__grains__, key, "_|-") - if ret != "_|-": - return ret - ret = salt.utils.data.traverse_dict_and_list(__pillar__, key, "_|-") - if ret != "_|-": - return ret - ret = salt.utils.data.traverse_dict_and_list( - __pillar__.get("master", {}), key, "_|-" - ) - if ret != "_|-": - return ret + if merge is None: + if not omit_opts: + ret = salt.utils.data.traverse_dict_and_list( + __opts__, key, "_|-", delimiter=delimiter + ) + if ret != "_|-": + return sdb.sdb_get(ret, __opts__) + + if not omit_grains: + ret = salt.utils.data.traverse_dict_and_list( + __grains__, key, "_|-", delimiter + ) + if ret != "_|-": + return sdb.sdb_get(ret, __opts__) + + if not omit_pillar: + ret = salt.utils.data.traverse_dict_and_list( + __pillar__, key, "_|-", delimiter=delimiter + ) + if ret != "_|-": + return sdb.sdb_get(ret, __opts__) + + if not omit_master: + ret = salt.utils.data.traverse_dict_and_list( + __pillar__.get("master", {}), key, "_|-", delimiter=delimiter + ) + if ret != "_|-": + return sdb.sdb_get(ret, __opts__) + + ret = salt.utils.data.traverse_dict_and_list( + DEFAULTS, key, "_|-", delimiter=delimiter + ) + if ret != "_|-": + return sdb.sdb_get(ret, __opts__) + else: + if merge not in ("recurse", "overwrite"): + log.warning( + "Unsupported merge strategy '%s'. Falling back to 'recurse'.", merge + ) + merge = "recurse" + + merge_lists = salt.config.master_config("/etc/salt/master").get( + "pillar_merge_lists" + ) + + data = copy.copy(DEFAULTS) + data = salt.utils.dictupdate.merge( + data, __pillar__.get("master", {}), strategy=merge, merge_lists=merge_lists + ) + data = salt.utils.dictupdate.merge( + data, __pillar__, strategy=merge, merge_lists=merge_lists + ) + data = salt.utils.dictupdate.merge( + data, __grains__, strategy=merge, merge_lists=merge_lists + ) + data = salt.utils.dictupdate.merge( + data, __opts__, strategy=merge, merge_lists=merge_lists + ) + ret = salt.utils.data.traverse_dict_and_list( + data, key, "_|-", delimiter=delimiter + ) + if ret != "_|-": + return sdb.sdb_get(ret, __opts__) + return default @@ -241,3 +504,19 @@ def dot_vals(value): if key.startswith(f"{value}."): ret[key] = val return ret + + +def items(): + """ + Return the complete config from the currently running minion process. + This includes defaults for values not set in the config file. + + CLI Example: + + .. code-block:: bash + + salt '*' config.items + """ + # This would otherwise be parsed as just the value of "local" in opts. + # In case the wfunc parsing is improved, this can be removed. + return {"local": {"return": __opts__.copy()}} diff --git a/tests/pytests/integration/ssh/test_config.py b/tests/pytests/integration/ssh/test_config.py new file mode 100644 index 00000000000..d3ae2b03a3e --- /dev/null +++ b/tests/pytests/integration/ssh/test_config.py @@ -0,0 +1,66 @@ +import pytest + +pytestmark = [pytest.mark.slow_test] + + +def test_items(salt_ssh_cli): + ret = salt_ssh_cli.run("config.items") + assert ret.returncode == 0 + assert isinstance(ret.data, dict) + assert "id" in ret.data + assert "grains" in ret.data + assert "__master_opts__" in ret.data + assert "cachedir" in ret.data + + +@pytest.mark.parametrize("omit", (False, True)) +def test_option_minion_opt(salt_ssh_cli, omit): + # Minion opt + ret = salt_ssh_cli.run("config.option", "id", omit_opts=omit, omit_grains=True) + assert ret.returncode == 0 + assert (ret.data != salt_ssh_cli.get_minion_tgt()) is omit + assert (ret.data == "") is omit + + +@pytest.mark.parametrize("omit", (False, True)) +def test_option_pillar(salt_ssh_cli, omit): + ret = salt_ssh_cli.run("config.option", "ext_spam", omit_pillar=omit) + assert ret.returncode == 0 + assert (ret.data != "eggs") is omit + assert (ret.data == "") is omit + + +@pytest.mark.parametrize("omit", (False, True)) +def test_option_grain(salt_ssh_cli, omit): + ret = salt_ssh_cli.run("config.option", "kernel", omit_grains=omit) + assert ret.returncode == 0 + assert ( + ret.data not in ("Darwin", "Linux", "FreeBSD", "OpenBSD", "Windows") + ) is omit + assert (ret.data == "") is omit + + +@pytest.mark.parametrize("omit", (False, True)) +def test_get_minion_opt(salt_ssh_cli, omit): + ret = salt_ssh_cli.run("config.get", "cachedir", omit_master=True, omit_opts=omit) + assert ret.returncode == 0 + assert (ret.data == "") is omit + assert ("minion" not in ret.data) is omit + + +@pytest.mark.parametrize("omit", (False, True)) +def test_get_pillar(salt_ssh_cli, omit): + ret = salt_ssh_cli.run("config.get", "ext_spam", omit_pillar=omit) + assert ret.returncode == 0 + assert (ret.data != "eggs") is omit + assert (ret.data == "") is omit + + +@pytest.mark.parametrize("omit", (False, True)) +def test_get_grain(salt_ssh_cli, omit): + ret = salt_ssh_cli.run("config.get", "kernel", omit_grains=omit) + assert ret.returncode == 0 + assert ( + ret.data not in ("Darwin", "Linux", "FreeBSD", "OpenBSD", "Windows") + ) is omit + assert (ret.data == "") is omit diff --git a/tests/pytests/unit/client/ssh/wrapper/test_config.py b/tests/pytests/unit/client/ssh/wrapper/test_config.py new file mode 100644 index 00000000000..64e89c762ad --- /dev/null +++ b/tests/pytests/unit/client/ssh/wrapper/test_config.py @@ -0,0 +1,219 @@ +""" + Taken 1:1 from test cases for salt.modules.config + This tests the SSH wrapper module. +""" + + +import fnmatch + +import pytest + +import salt.client.ssh.wrapper.config as config +from tests.support.mock import patch + + +@pytest.fixture +def defaults(): + return { + "test.option.foo": "value of test.option.foo in defaults", + "test.option.bar": "value of test.option.bar in defaults", + "test.option.baz": "value of test.option.baz in defaults", + "test.option": "value of test.option in defaults", + } + + +@pytest.fixture +def no_match(): + return "test.option.nope" + + +@pytest.fixture +def opt_name(): + return "test.option.foo" + + +@pytest.fixture +def wildcard_opt_name(): + return "test.option.b*" + + +@pytest.fixture +def configure_loader_modules(): + return { + config: { + "__opts__": { + "test.option.foo": "value of test.option.foo in __opts__", + "test.option.bar": "value of test.option.bar in __opts__", + "test.option.baz": "value of test.option.baz in __opts__", + }, + "__pillar__": { + "test.option.foo": "value of test.option.foo in __pillar__", + "test.option.bar": "value of test.option.bar in __pillar__", + "test.option.baz": "value of test.option.baz in __pillar__", + "master": { + "test.option.foo": "value of test.option.foo in master", + "test.option.bar": "value of test.option.bar in master", + "test.option.baz": "value of test.option.baz in master", + }, + }, + "__grains__": { + "test.option.foo": "value of test.option.foo in __grains__", + "test.option.bar": "value of test.option.bar in __grains__", + "test.option.baz": "value of test.option.baz in __grains__", + }, + } + } + + +def _wildcard_match(data, wildcard_opt_name): + return {x: data[x] for x in fnmatch.filter(data, wildcard_opt_name)} + + +def test_defaults_only_name(defaults): + with patch.dict(config.DEFAULTS, defaults): + opt_name = "test.option" + opt = config.option(opt_name) + assert opt == config.DEFAULTS[opt_name] + + +def test_no_match(defaults, no_match, wildcard_opt_name): + """ + Make sure that the defa + """ + with patch.dict(config.DEFAULTS, defaults): + ret = config.option(no_match) + assert ret == "", ret + + default = "wat" + ret = config.option(no_match, default=default) + assert ret == default, ret + + ret = config.option(no_match, wildcard=True) + assert ret == {}, ret + + default = {"foo": "bar"} + ret = config.option(no_match, default=default, wildcard=True) + assert ret == default, ret + + # Should be no match since wildcard=False + ret = config.option(wildcard_opt_name) + assert ret == "", ret + + +def test_omits(defaults, opt_name, wildcard_opt_name): + with patch.dict(config.DEFAULTS, defaults): + + # ********** OMIT NOTHING ********** + + # Match should be in __opts__ dict + ret = config.option(opt_name) + assert ret == config.__opts__[opt_name], ret + + # Wildcard match + ret = config.option(wildcard_opt_name, wildcard=True) + assert ret == _wildcard_match(config.__opts__, wildcard_opt_name), ret + + # ********** OMIT __opts__ ********** + + # Match should be in __grains__ dict + ret = config.option(opt_name, omit_opts=True) + assert ret == config.__grains__[opt_name], ret + + # Wildcard match + ret = config.option(wildcard_opt_name, omit_opts=True, wildcard=True) + assert ret == _wildcard_match(config.__grains__, wildcard_opt_name), ret + + # ********** OMIT __opts__, __grains__ ********** + + # Match should be in __pillar__ dict + ret = config.option(opt_name, omit_opts=True, omit_grains=True) + assert ret == config.__pillar__[opt_name], ret + + # Wildcard match + ret = config.option( + wildcard_opt_name, omit_opts=True, omit_grains=True, wildcard=True + ) + assert ret == _wildcard_match(config.__pillar__, wildcard_opt_name), ret + + # ********** OMIT __opts__, __grains__, __pillar__ ********** + + # Match should be in master opts + ret = config.option( + opt_name, omit_opts=True, omit_grains=True, omit_pillar=True + ) + assert ret == config.__pillar__["master"][opt_name], ret + + # Wildcard match + ret = config.option( + wildcard_opt_name, + omit_opts=True, + omit_grains=True, + omit_pillar=True, + wildcard=True, + ) + assert ret == _wildcard_match( + config.__pillar__["master"], wildcard_opt_name + ), ret + + # ********** OMIT ALL THE THINGS ********** + + # Match should be in master opts + ret = config.option( + opt_name, + omit_opts=True, + omit_grains=True, + omit_pillar=True, + omit_master=True, + ) + assert ret == config.DEFAULTS[opt_name], ret + + # Wildcard match + ret = config.option( + wildcard_opt_name, + omit_opts=True, + omit_grains=True, + omit_pillar=True, + omit_master=True, + wildcard=True, + ) + assert ret == _wildcard_match(config.DEFAULTS, wildcard_opt_name), ret + + # Match should be in master opts + ret = config.option(opt_name, omit_all=True) + assert ret == config.DEFAULTS[opt_name], ret + + # Wildcard match + ret = config.option(wildcard_opt_name, omit_all=True, wildcard=True) + assert ret == _wildcard_match(config.DEFAULTS, wildcard_opt_name), ret + + +# --- Additional tests not found in the execution module tests + + +@pytest.mark.parametrize("backup", ("", "minion", "master", "both")) +def test_backup_mode(backup): + res = config.backup_mode(backup) + assert res == backup or "minion" + + +@pytest.mark.parametrize( + "uri,expected", + (("salt://my/foo.txt", True), ("mysql://foo:bar@foo.bar/baz", False)), +) +def test_valid_fileproto(uri, expected): + res = config.valid_fileproto(uri) + assert res is expected + + +def test_dot_vals(): + extra_master_opt = ("test.option.baah", "value of test.option.baah in master") + with patch.dict(config.__pillar__, {"master": dict((extra_master_opt,))}): + res = config.dot_vals("test") + assert isinstance(res, dict) + assert res + for var in ("foo", "bar", "baz"): + key = f"test.option.{var}" + assert key in res + assert res[key] == f"value of test.option.{var} in __opts__" + assert extra_master_opt[0] in res + assert res[extra_master_opt[0]] == extra_master_opt[1]