Sync config SSH wrapper with execution module

The wrapper has diverged significantly from the module.

* `option` did not check grains
* `option` did not have `omit_all` and `wildcard` parameters
* `get` missed several parameters: `delimiter`, `merge` and all `omit_*`
* There was no wrapping function for `items`.
This commit is contained in:
jeanluc 2023-11-06 12:42:55 +01:00
parent 82f90e2f15
commit 8356be888b
No known key found for this signature in database
GPG key ID: 3EB52D4C754CD898
4 changed files with 613 additions and 48 deletions

1
changelog/56441.fixed.md Normal file
View file

@ -0,0 +1 @@
Fixed config.get does not support merge option with salt-ssh

View file

@ -2,17 +2,22 @@
Return config information Return config information
""" """
import copy
import fnmatch
import logging
import os import os
import re import urllib.parse
import salt.syspaths as syspaths import salt.syspaths as syspaths
import salt.utils.data import salt.utils.data
import salt.utils.files import salt.utils.files
import salt.utils.sdb as sdb
log = logging.getLogger(__name__)
# Set up the default values for all systems # Set up the default values for all systems
DEFAULTS = { DEFAULTS = {
"mongo.db": "salt", "mongo.db": "salt",
"mongo.host": "salt",
"mongo.password": "", "mongo.password": "",
"mongo.port": 27017, "mongo.port": 27017,
"mongo.user": "", "mongo.user": "",
@ -38,9 +43,12 @@ DEFAULTS = {
"solr.num_backups": 1, "solr.num_backups": 1,
"poudriere.config": "/usr/local/etc/poudriere.conf", "poudriere.config": "/usr/local/etc/poudriere.conf",
"poudriere.config_dir": "/usr/local/etc/poudriere.d", "poudriere.config_dir": "/usr/local/etc/poudriere.d",
"ldap.uri": "",
"ldap.server": "localhost", "ldap.server": "localhost",
"ldap.port": "389", "ldap.port": "389",
"ldap.tls": False, "ldap.tls": False,
"ldap.no_verify": False,
"ldap.anonymous": True,
"ldap.scope": 2, "ldap.scope": 2,
"ldap.attrs": None, "ldap.attrs": None,
"ldap.binddn": "", "ldap.binddn": "",
@ -51,6 +59,11 @@ DEFAULTS = {
"tunnel": False, "tunnel": False,
"images": os.path.join(syspaths.SRV_ROOT_DIR, "salt-images"), "images": os.path.join(syspaths.SRV_ROOT_DIR, "salt-images"),
}, },
"docker.exec_driver": "docker-exec",
"docker.compare_container_networks": {
"static": ["Aliases", "Links", "IPAMConfig"],
"automatic": ["IPAddress", "Gateway", "GlobalIPv6Address", "IPv6Gateway"],
},
} }
@ -96,15 +109,66 @@ def valid_fileproto(uri):
salt '*' config.valid_fileproto salt://path/to/file salt '*' config.valid_fileproto salt://path/to/file
""" """
try: return urllib.parse.urlparse(uri).scheme in salt.utils.files.VALID_PROTOS
return bool(re.match("^(?:salt|https?|ftp)://", uri))
except Exception: # pylint: disable=broad-except
return False
def option(value, default="", omit_opts=False, omit_master=False, omit_pillar=False): def option(
value,
default=None,
omit_opts=False,
omit_grains=False,
omit_pillar=False,
omit_master=False,
omit_all=False,
wildcard=False,
):
""" """
Pass in a generic option and receive the value that will be assigned Returns the setting for the specified config value. The priority for
matches is the same as in :py:func:`config.get <salt.modules.config.get>`,
only this function does not recurse into nested data structures. Another
difference between this function and :py:func:`config.get
<salt.modules.config.get>` is that it comes with a set of "sane defaults".
To view these, you can run the following command:
.. code-block:: bash
salt '*' config.option '*' omit_all=True wildcard=True
default
The default value if no match is found. If not specified, then the
fallback default will be an empty string, unless ``wildcard=True``, in
which case the return will be an empty dictionary.
omit_opts : False
Pass as ``True`` to exclude matches from the minion configuration file
omit_grains : False
Pass as ``True`` to exclude matches from the grains
omit_pillar : False
Pass as ``True`` to exclude matches from the pillar data
omit_master : False
Pass as ``True`` to exclude matches from the master configuration file
omit_all : True
Shorthand to omit all of the above and return matches only from the
"sane defaults".
.. versionadded:: 3000
wildcard : False
If used, this will perform pattern matching on keys. Note that this
will also significantly change the return data. Instead of only a value
being returned, a dictionary mapping the matched keys to their values
is returned. For example, using ``wildcard=True`` with a ``key`` of
``'foo.ba*`` could return a dictionary like so:
.. code-block:: python
{'foo.bar': True, 'foo.baz': False}
.. versionadded:: 3000
CLI Example: CLI Example:
@ -112,18 +176,48 @@ def option(value, default="", omit_opts=False, omit_master=False, omit_pillar=Fa
salt '*' config.option redis.host salt '*' config.option redis.host
""" """
if not omit_opts: if omit_all:
if value in __opts__: omit_opts = omit_grains = omit_pillar = omit_master = True
return __opts__[value]
if not omit_master: if default is None:
if value in __pillar__.get("master", {}): default = "" if not wildcard else {}
return __pillar__["master"][value]
if not omit_pillar: if not wildcard:
if value in __pillar__: if not omit_opts:
return __pillar__[value] if value in __opts__:
if value in DEFAULTS: return __opts__[value]
return DEFAULTS[value] if not omit_grains:
return default if value in __grains__:
return __grains__[value]
if not omit_pillar:
if value in __pillar__:
return __pillar__[value]
if not omit_master:
if value in __pillar__.get("master", {}):
return __pillar__["master"][value]
if value in DEFAULTS:
return DEFAULTS[value]
# No match
return default
else:
# We need to do the checks in the reverse order so that minion opts
# takes precedence
ret = {}
for omit, data in (
(omit_master, __pillar__.get("master", {})),
(omit_pillar, __pillar__),
(omit_grains, __grains__),
(omit_opts, __opts__),
):
if not omit:
ret.update({x: data[x] for x in fnmatch.filter(data, value)})
# Check the DEFAULTS as well to see if the pattern matches it
for item in (x for x in fnmatch.filter(DEFAULTS, value) if x not in ret):
ret[item] = DEFAULTS[item]
# If no matches, return the default
return ret or default
def merge(value, default="", omit_opts=False, omit_master=False, omit_pillar=False): def merge(value, default="", omit_opts=False, omit_master=False, omit_pillar=False):
@ -171,54 +265,223 @@ def merge(value, default="", omit_opts=False, omit_master=False, omit_pillar=Fal
ret = list(ret) + list(tmp) ret = list(ret) + list(tmp)
if ret is None and value in DEFAULTS: if ret is None and value in DEFAULTS:
return DEFAULTS[value] return DEFAULTS[value]
return ret or default if ret is None:
return default
return ret
def get(key, default=""): def get(
key,
default="",
delimiter=":",
merge=None,
omit_opts=False,
omit_pillar=False,
omit_master=False,
omit_grains=False,
):
""" """
.. versionadded:: 0.14.0 .. versionadded:: 0.14.0
Attempt to retrieve the named value from opts, pillar, grains of the master Attempt to retrieve the named value from the minion config file, pillar,
config, if the named value is not available return the passed default. grains or the master config. If the named value is not available, return
The default return is an empty string. the value specified by the ``default`` argument. If this argument is not
specified, ``default`` falls back to an empty string.
The value can also represent a value in a nested dict using a ":" delimiter Values can also be retrieved from nested dictionaries. Assume the below
for the dict. This means that if a dict looks like this:: data structure:
.. code-block:: python
{'pkg': {'apache': 'httpd'}} {'pkg': {'apache': 'httpd'}}
To retrieve the value associated with the apache key in the pkg dict this To retrieve the value associated with the ``apache`` key, in the
key can be passed:: sub-dictionary corresponding to the ``pkg`` key, the following command can
be used:
pkg:apache .. code-block:: bash
This routine traverses these data stores in this order: salt myminion config.get pkg:apache
- Local minion config (opts) The ``:`` (colon) is used to represent a nested dictionary level.
.. versionchanged:: 2015.5.0
The ``delimiter`` argument was added, to allow delimiters other than
``:`` to be used.
This function traverses these data stores in this order, returning the
first match found:
- Minion configuration
- Minion's grains - Minion's grains
- Minion's pillar - Minion's pillar data
- Master config - Master configuration (requires :conf_minion:`pillar_opts` to be set to
``True`` in Minion config file in order to work)
This means that if there is a value that is going to be the same for the
majority of minions, it can be configured in the Master config file, and
then overridden using the grains, pillar, or Minion config file.
Adding config options to the Master or Minion configuration file is easy:
.. code-block:: yaml
my-config-option: value
cafe-menu:
- egg and bacon
- egg sausage and bacon
- egg and spam
- egg bacon and spam
- egg bacon sausage and spam
- spam bacon sausage and spam
- spam egg spam spam bacon and spam
- spam sausage spam spam bacon spam tomato and spam
.. note::
Minion configuration options built into Salt (like those defined
:ref:`here <configuration-salt-minion>`) will *always* be defined in
the Minion configuration and thus *cannot be overridden by grains or
pillar data*. However, additional (user-defined) configuration options
(as in the above example) will not be in the Minion configuration by
default and thus can be overridden using grains/pillar data by leaving
the option out of the minion config file.
**Arguments**
delimiter
.. versionadded:: 2015.5.0
Override the delimiter used to separate nested levels of a data
structure.
merge
.. versionadded:: 2015.5.0
If passed, this parameter will change the behavior of the function so
that, instead of traversing each data store above in order and
returning the first match, the data stores are first merged together
and then searched. The pillar data is merged into the master config
data, then the grains are merged, followed by the Minion config data.
The resulting data structure is then searched for a match. This allows
for configurations to be more flexible.
.. note::
The merging described above does not mean that grain data will end
up in the Minion's pillar data, or pillar data will end up in the
master config data, etc. The data is just combined for the purposes
of searching an amalgam of the different data stores.
The supported merge strategies are as follows:
- **recurse** - If a key exists in both dictionaries, and the new value
is not a dictionary, it is replaced. Otherwise, the sub-dictionaries
are merged together into a single dictionary, recursively on down,
following the same criteria. For example:
.. code-block:: python
>>> dict1 = {'foo': {'bar': 1, 'qux': True},
'hosts': ['a', 'b', 'c'],
'only_x': None}
>>> dict2 = {'foo': {'baz': 2, 'qux': False},
'hosts': ['d', 'e', 'f'],
'only_y': None}
>>> merged
{'foo': {'bar': 1, 'baz': 2, 'qux': False},
'hosts': ['d', 'e', 'f'],
'only_dict1': None,
'only_dict2': None}
- **overwrite** - If a key exists in the top level of both
dictionaries, the new value completely overwrites the old. For
example:
.. code-block:: python
>>> dict1 = {'foo': {'bar': 1, 'qux': True},
'hosts': ['a', 'b', 'c'],
'only_x': None}
>>> dict2 = {'foo': {'baz': 2, 'qux': False},
'hosts': ['d', 'e', 'f'],
'only_y': None}
>>> merged
{'foo': {'baz': 2, 'qux': False},
'hosts': ['d', 'e', 'f'],
'only_dict1': None,
'only_dict2': None}
CLI Example: CLI Example:
.. code-block:: bash .. code-block:: bash
salt '*' config.get pkg:apache salt '*' config.get pkg:apache
salt '*' config.get lxc.container_profile:centos merge=recurse
""" """
ret = salt.utils.data.traverse_dict_and_list(__opts__, key, "_|-") if merge is None:
if ret != "_|-": if not omit_opts:
return ret ret = salt.utils.data.traverse_dict_and_list(
ret = salt.utils.data.traverse_dict_and_list(__grains__, key, "_|-") __opts__, key, "_|-", delimiter=delimiter
if ret != "_|-": )
return ret if ret != "_|-":
ret = salt.utils.data.traverse_dict_and_list(__pillar__, key, "_|-") return sdb.sdb_get(ret, __opts__)
if ret != "_|-":
return ret if not omit_grains:
ret = salt.utils.data.traverse_dict_and_list( ret = salt.utils.data.traverse_dict_and_list(
__pillar__.get("master", {}), key, "_|-" __grains__, key, "_|-", delimiter
) )
if ret != "_|-": if ret != "_|-":
return ret return sdb.sdb_get(ret, __opts__)
if not omit_pillar:
ret = salt.utils.data.traverse_dict_and_list(
__pillar__, key, "_|-", delimiter=delimiter
)
if ret != "_|-":
return sdb.sdb_get(ret, __opts__)
if not omit_master:
ret = salt.utils.data.traverse_dict_and_list(
__pillar__.get("master", {}), key, "_|-", delimiter=delimiter
)
if ret != "_|-":
return sdb.sdb_get(ret, __opts__)
ret = salt.utils.data.traverse_dict_and_list(
DEFAULTS, key, "_|-", delimiter=delimiter
)
if ret != "_|-":
return sdb.sdb_get(ret, __opts__)
else:
if merge not in ("recurse", "overwrite"):
log.warning(
"Unsupported merge strategy '%s'. Falling back to 'recurse'.", merge
)
merge = "recurse"
merge_lists = salt.config.master_config("/etc/salt/master").get(
"pillar_merge_lists"
)
data = copy.copy(DEFAULTS)
data = salt.utils.dictupdate.merge(
data, __pillar__.get("master", {}), strategy=merge, merge_lists=merge_lists
)
data = salt.utils.dictupdate.merge(
data, __pillar__, strategy=merge, merge_lists=merge_lists
)
data = salt.utils.dictupdate.merge(
data, __grains__, strategy=merge, merge_lists=merge_lists
)
data = salt.utils.dictupdate.merge(
data, __opts__, strategy=merge, merge_lists=merge_lists
)
ret = salt.utils.data.traverse_dict_and_list(
data, key, "_|-", delimiter=delimiter
)
if ret != "_|-":
return sdb.sdb_get(ret, __opts__)
return default return default
@ -241,3 +504,19 @@ def dot_vals(value):
if key.startswith("{}.".format(value)): if key.startswith("{}.".format(value)):
ret[key] = val ret[key] = val
return ret return ret
def items():
"""
Return the complete config from the currently running minion process.
This includes defaults for values not set in the config file.
CLI Example:
.. code-block:: bash
salt '*' config.items
"""
# This would otherwise be parsed as just the value of "local" in opts.
# In case the wfunc parsing is improved, this can be removed.
return {"local": {"return": __opts__.copy()}}

View file

@ -0,0 +1,66 @@
import pytest
pytestmark = [pytest.mark.slow_test]
def test_items(salt_ssh_cli):
ret = salt_ssh_cli.run("config.items")
assert ret.returncode == 0
assert isinstance(ret.data, dict)
assert "id" in ret.data
assert "grains" in ret.data
assert "__master_opts__" in ret.data
assert "cachedir" in ret.data
@pytest.mark.parametrize("omit", (False, True))
def test_option_minion_opt(salt_ssh_cli, omit):
# Minion opt
ret = salt_ssh_cli.run("config.option", "id", omit_opts=omit, omit_grains=True)
assert ret.returncode == 0
assert (ret.data != salt_ssh_cli.get_minion_tgt()) is omit
assert (ret.data == "") is omit
@pytest.mark.parametrize("omit", (False, True))
def test_option_pillar(salt_ssh_cli, omit):
ret = salt_ssh_cli.run("config.option", "ext_spam", omit_pillar=omit)
assert ret.returncode == 0
assert (ret.data != "eggs") is omit
assert (ret.data == "") is omit
@pytest.mark.parametrize("omit", (False, True))
def test_option_grain(salt_ssh_cli, omit):
ret = salt_ssh_cli.run("config.option", "kernel", omit_grains=omit)
assert ret.returncode == 0
assert (
ret.data not in ("Darwin", "Linux", "FreeBSD", "OpenBSD", "Windows")
) is omit
assert (ret.data == "") is omit
@pytest.mark.parametrize("omit", (False, True))
def test_get_minion_opt(salt_ssh_cli, omit):
ret = salt_ssh_cli.run("config.get", "cachedir", omit_master=True, omit_opts=omit)
assert ret.returncode == 0
assert (ret.data == "") is omit
assert ("minion" not in ret.data) is omit
@pytest.mark.parametrize("omit", (False, True))
def test_get_pillar(salt_ssh_cli, omit):
ret = salt_ssh_cli.run("config.get", "ext_spam", omit_pillar=omit)
assert ret.returncode == 0
assert (ret.data != "eggs") is omit
assert (ret.data == "") is omit
@pytest.mark.parametrize("omit", (False, True))
def test_get_grain(salt_ssh_cli, omit):
ret = salt_ssh_cli.run("config.get", "kernel", omit_grains=omit)
assert ret.returncode == 0
assert (
ret.data not in ("Darwin", "Linux", "FreeBSD", "OpenBSD", "Windows")
) is omit
assert (ret.data == "") is omit

View file

@ -0,0 +1,219 @@
"""
Taken 1:1 from test cases for salt.modules.config
This tests the SSH wrapper module.
"""
import fnmatch
import pytest
import salt.client.ssh.wrapper.config as config
from tests.support.mock import patch
@pytest.fixture
def defaults():
return {
"test.option.foo": "value of test.option.foo in defaults",
"test.option.bar": "value of test.option.bar in defaults",
"test.option.baz": "value of test.option.baz in defaults",
"test.option": "value of test.option in defaults",
}
@pytest.fixture
def no_match():
return "test.option.nope"
@pytest.fixture
def opt_name():
return "test.option.foo"
@pytest.fixture
def wildcard_opt_name():
return "test.option.b*"
@pytest.fixture
def configure_loader_modules():
return {
config: {
"__opts__": {
"test.option.foo": "value of test.option.foo in __opts__",
"test.option.bar": "value of test.option.bar in __opts__",
"test.option.baz": "value of test.option.baz in __opts__",
},
"__pillar__": {
"test.option.foo": "value of test.option.foo in __pillar__",
"test.option.bar": "value of test.option.bar in __pillar__",
"test.option.baz": "value of test.option.baz in __pillar__",
"master": {
"test.option.foo": "value of test.option.foo in master",
"test.option.bar": "value of test.option.bar in master",
"test.option.baz": "value of test.option.baz in master",
},
},
"__grains__": {
"test.option.foo": "value of test.option.foo in __grains__",
"test.option.bar": "value of test.option.bar in __grains__",
"test.option.baz": "value of test.option.baz in __grains__",
},
}
}
def _wildcard_match(data, wildcard_opt_name):
return {x: data[x] for x in fnmatch.filter(data, wildcard_opt_name)}
def test_defaults_only_name(defaults):
with patch.dict(config.DEFAULTS, defaults):
opt_name = "test.option"
opt = config.option(opt_name)
assert opt == config.DEFAULTS[opt_name]
def test_no_match(defaults, no_match, wildcard_opt_name):
"""
Make sure that the defa
"""
with patch.dict(config.DEFAULTS, defaults):
ret = config.option(no_match)
assert ret == "", ret
default = "wat"
ret = config.option(no_match, default=default)
assert ret == default, ret
ret = config.option(no_match, wildcard=True)
assert ret == {}, ret
default = {"foo": "bar"}
ret = config.option(no_match, default=default, wildcard=True)
assert ret == default, ret
# Should be no match since wildcard=False
ret = config.option(wildcard_opt_name)
assert ret == "", ret
def test_omits(defaults, opt_name, wildcard_opt_name):
with patch.dict(config.DEFAULTS, defaults):
# ********** OMIT NOTHING **********
# Match should be in __opts__ dict
ret = config.option(opt_name)
assert ret == config.__opts__[opt_name], ret
# Wildcard match
ret = config.option(wildcard_opt_name, wildcard=True)
assert ret == _wildcard_match(config.__opts__, wildcard_opt_name), ret
# ********** OMIT __opts__ **********
# Match should be in __grains__ dict
ret = config.option(opt_name, omit_opts=True)
assert ret == config.__grains__[opt_name], ret
# Wildcard match
ret = config.option(wildcard_opt_name, omit_opts=True, wildcard=True)
assert ret == _wildcard_match(config.__grains__, wildcard_opt_name), ret
# ********** OMIT __opts__, __grains__ **********
# Match should be in __pillar__ dict
ret = config.option(opt_name, omit_opts=True, omit_grains=True)
assert ret == config.__pillar__[opt_name], ret
# Wildcard match
ret = config.option(
wildcard_opt_name, omit_opts=True, omit_grains=True, wildcard=True
)
assert ret == _wildcard_match(config.__pillar__, wildcard_opt_name), ret
# ********** OMIT __opts__, __grains__, __pillar__ **********
# Match should be in master opts
ret = config.option(
opt_name, omit_opts=True, omit_grains=True, omit_pillar=True
)
assert ret == config.__pillar__["master"][opt_name], ret
# Wildcard match
ret = config.option(
wildcard_opt_name,
omit_opts=True,
omit_grains=True,
omit_pillar=True,
wildcard=True,
)
assert ret == _wildcard_match(
config.__pillar__["master"], wildcard_opt_name
), ret
# ********** OMIT ALL THE THINGS **********
# Match should be in master opts
ret = config.option(
opt_name,
omit_opts=True,
omit_grains=True,
omit_pillar=True,
omit_master=True,
)
assert ret == config.DEFAULTS[opt_name], ret
# Wildcard match
ret = config.option(
wildcard_opt_name,
omit_opts=True,
omit_grains=True,
omit_pillar=True,
omit_master=True,
wildcard=True,
)
assert ret == _wildcard_match(config.DEFAULTS, wildcard_opt_name), ret
# Match should be in master opts
ret = config.option(opt_name, omit_all=True)
assert ret == config.DEFAULTS[opt_name], ret
# Wildcard match
ret = config.option(wildcard_opt_name, omit_all=True, wildcard=True)
assert ret == _wildcard_match(config.DEFAULTS, wildcard_opt_name), ret
# --- Additional tests not found in the execution module tests
@pytest.mark.parametrize("backup", ("", "minion", "master", "both"))
def test_backup_mode(backup):
res = config.backup_mode(backup)
assert res == backup or "minion"
@pytest.mark.parametrize(
"uri,expected",
(("salt://my/foo.txt", True), ("mysql://foo:bar@foo.bar/baz", False)),
)
def test_valid_fileproto(uri, expected):
res = config.valid_fileproto(uri)
assert res is expected
def test_dot_vals():
extra_master_opt = ("test.option.baah", "value of test.option.baah in master")
with patch.dict(config.__pillar__, {"master": dict((extra_master_opt,))}):
res = config.dot_vals("test")
assert isinstance(res, dict)
assert res
for var in ("foo", "bar", "baz"):
key = f"test.option.{var}"
assert key in res
assert res[key] == f"value of test.option.{var} in __opts__"
assert extra_master_opt[0] in res
assert res[extra_master_opt[0]] == extra_master_opt[1]