mirror of
https://github.com/saltstack/salt.git
synced 2025-04-17 10:10:20 +00:00
Move vault parameter list expansion to utils module
This commit is contained in:
jeanluc
parent
a1a07d1c8c
commit
7f9d276c2a
4 changed files with 132 additions and 128 deletions
|
|
@ -10,7 +10,6 @@ documented in the execution module docs.
|
|||
import base64
|
||||
import json
|
||||
import logging
|
||||
import string
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
|
@ -211,7 +210,9 @@ def _get_policies(minion_id, config):
|
|||
policies = []
|
||||
for pattern in policy_patterns:
|
||||
try:
|
||||
for expanded_pattern in _expand_pattern_lists(pattern, **mappings):
|
||||
for expanded_pattern in __utils__["vault.expand_pattern_lists"](
|
||||
pattern, **mappings
|
||||
):
|
||||
policies.append(
|
||||
expanded_pattern.format(**mappings).lower() # Vault requirement
|
||||
)
|
||||
|
|
@ -222,56 +223,6 @@ def _get_policies(minion_id, config):
|
|||
return policies
|
||||
|
||||
|
||||
def _expand_pattern_lists(pattern, **mappings):
|
||||
"""
|
||||
Expands the pattern for any list-valued mappings, such that for any list of
|
||||
length N in the mappings present in the pattern, N copies of the pattern are
|
||||
returned, each with an element of the list substituted.
|
||||
|
||||
pattern:
|
||||
A pattern to expand, for example ``by-role/{grains[roles]}``
|
||||
|
||||
mappings:
|
||||
A dictionary of variables that can be expanded into the pattern.
|
||||
|
||||
Example: Given the pattern `` by-role/{grains[roles]}`` and the below grains
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
grains:
|
||||
roles:
|
||||
- web
|
||||
- database
|
||||
|
||||
This function will expand into two patterns,
|
||||
``[by-role/web, by-role/database]``.
|
||||
|
||||
Note that this method does not expand any non-list patterns.
|
||||
"""
|
||||
expanded_patterns = []
|
||||
f = string.Formatter()
|
||||
|
||||
# This function uses a string.Formatter to get all the formatting tokens from
|
||||
# the pattern, then recursively replaces tokens whose expanded value is a
|
||||
# list. For a list with N items, it will create N new pattern strings and
|
||||
# then continue with the next token. In practice this is expected to not be
|
||||
# very expensive, since patterns will typically involve a handful of lists at
|
||||
# most.
|
||||
|
||||
for (_, field_name, _, _) in f.parse(pattern):
|
||||
if field_name is None:
|
||||
continue
|
||||
(value, _) = f.get_field(field_name, None, mappings)
|
||||
if isinstance(value, list):
|
||||
token = "{{{0}}}".format(field_name)
|
||||
expanded = [pattern.replace(token, str(elem)) for elem in value]
|
||||
for expanded_item in expanded:
|
||||
result = _expand_pattern_lists(expanded_item, **mappings)
|
||||
expanded_patterns += result
|
||||
return expanded_patterns
|
||||
return [pattern]
|
||||
|
||||
|
||||
def _selftoken_expired():
|
||||
"""
|
||||
Validate the current token exists and is still valid
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ documented in the execution module docs.
|
|||
import base64
|
||||
import logging
|
||||
import os
|
||||
import string
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
|
|
@ -545,3 +546,53 @@ def _get_secret_path_metadata(path):
|
|||
except Exception as err: # pylint: disable=broad-except
|
||||
log.error("Failed to get secret metadata %s: %s", type(err).__name__, err)
|
||||
return ret
|
||||
|
||||
|
||||
def expand_pattern_lists(pattern, **mappings):
|
||||
"""
|
||||
Expands the pattern for any list-valued mappings, such that for any list of
|
||||
length N in the mappings present in the pattern, N copies of the pattern are
|
||||
returned, each with an element of the list substituted.
|
||||
|
||||
pattern:
|
||||
A pattern to expand, for example ``by-role/{grains[roles]}``
|
||||
|
||||
mappings:
|
||||
A dictionary of variables that can be expanded into the pattern.
|
||||
|
||||
Example: Given the pattern `` by-role/{grains[roles]}`` and the below grains
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
grains:
|
||||
roles:
|
||||
- web
|
||||
- database
|
||||
|
||||
This function will expand into two patterns,
|
||||
``[by-role/web, by-role/database]``.
|
||||
|
||||
Note that this method does not expand any non-list patterns.
|
||||
"""
|
||||
expanded_patterns = []
|
||||
f = string.Formatter()
|
||||
|
||||
# This function uses a string.Formatter to get all the formatting tokens from
|
||||
# the pattern, then recursively replaces tokens whose expanded value is a
|
||||
# list. For a list with N items, it will create N new pattern strings and
|
||||
# then continue with the next token. In practice this is expected to not be
|
||||
# very expensive, since patterns will typically involve a handful of lists at
|
||||
# most.
|
||||
|
||||
for (_, field_name, _, _) in f.parse(pattern):
|
||||
if field_name is None:
|
||||
continue
|
||||
(value, _) = f.get_field(field_name, None, mappings)
|
||||
if isinstance(value, list):
|
||||
token = "{{{0}}}".format(field_name)
|
||||
expanded = [pattern.replace(token, str(elem)) for elem in value]
|
||||
for expanded_item in expanded:
|
||||
result = expand_pattern_lists(expanded_item, **mappings)
|
||||
expanded_patterns += result
|
||||
return expanded_patterns
|
||||
return [pattern]
|
||||
|
|
|
|||
|
|
@ -544,3 +544,42 @@ def test_get_secret_path_metadata_no_cache(metadata_v2, cache_uses, cache_secret
|
|||
assert function_result == metadata_v2
|
||||
mock_write_cache.assert_called_with(cache_object)
|
||||
assert cache_object == expected_cache_object
|
||||
|
||||
|
||||
def test_expand_pattern_lists():
|
||||
"""
|
||||
Ensure expand_pattern_lists works as intended:
|
||||
- Expand list-valued patterns
|
||||
- Do not change non-list-valued tokens
|
||||
"""
|
||||
cases = {
|
||||
"no-tokens-to-replace": ["no-tokens-to-replace"],
|
||||
"single-dict:{minion}": ["single-dict:{minion}"],
|
||||
"single-list:{grains[roles]}": ["single-list:web", "single-list:database"],
|
||||
"multiple-lists:{grains[roles]}+{grains[aux]}": [
|
||||
"multiple-lists:web+foo",
|
||||
"multiple-lists:web+bar",
|
||||
"multiple-lists:database+foo",
|
||||
"multiple-lists:database+bar",
|
||||
],
|
||||
"single-list-with-dicts:{grains[id]}+{grains[roles]}+{grains[id]}": [
|
||||
"single-list-with-dicts:{grains[id]}+web+{grains[id]}",
|
||||
"single-list-with-dicts:{grains[id]}+database+{grains[id]}",
|
||||
],
|
||||
"deeply-nested-list:{grains[deep][foo][bar][baz]}": [
|
||||
"deeply-nested-list:hello",
|
||||
"deeply-nested-list:world",
|
||||
],
|
||||
}
|
||||
|
||||
pattern_vars = {
|
||||
"id": "test-minion",
|
||||
"roles": ["web", "database"],
|
||||
"aux": ["foo", "bar"],
|
||||
"deep": {"foo": {"bar": {"baz": ["hello", "world"]}}},
|
||||
}
|
||||
|
||||
mappings = {"minion": "test-minion", "grains": pattern_vars}
|
||||
for case, correct_output in cases.items():
|
||||
output = vault.expand_pattern_lists(case, **mappings)
|
||||
assert output == correct_output
|
||||
|
|
|
|||
|
|
@ -33,91 +33,46 @@ class VaultTest(TestCase, LoaderModuleMockMixin):
|
|||
def tearDown(self):
|
||||
del self.grains
|
||||
|
||||
def test_pattern_list_expander(self):
|
||||
"""
|
||||
Ensure _expand_pattern_lists works as intended:
|
||||
- Expand list-valued patterns
|
||||
- Do not change non-list-valued tokens
|
||||
"""
|
||||
cases = {
|
||||
"no-tokens-to-replace": ["no-tokens-to-replace"],
|
||||
"single-dict:{minion}": ["single-dict:{minion}"],
|
||||
"single-list:{grains[roles]}": ["single-list:web", "single-list:database"],
|
||||
"multiple-lists:{grains[roles]}+{grains[aux]}": [
|
||||
"multiple-lists:web+foo",
|
||||
"multiple-lists:web+bar",
|
||||
"multiple-lists:database+foo",
|
||||
"multiple-lists:database+bar",
|
||||
],
|
||||
"single-list-with-dicts:{grains[id]}+{grains[roles]}+{grains[id]}": [
|
||||
"single-list-with-dicts:{grains[id]}+web+{grains[id]}",
|
||||
"single-list-with-dicts:{grains[id]}+database+{grains[id]}",
|
||||
],
|
||||
"deeply-nested-list:{grains[deep][foo][bar][baz]}": [
|
||||
"deeply-nested-list:hello",
|
||||
"deeply-nested-list:world",
|
||||
],
|
||||
}
|
||||
|
||||
# The mappings dict is assembled in _get_policies, so emulate here
|
||||
mappings = {"minion": self.grains["id"], "grains": self.grains}
|
||||
for case, correct_output in cases.items():
|
||||
output = vault._expand_pattern_lists(
|
||||
case, **mappings
|
||||
) # pylint: disable=protected-access
|
||||
diff = set(output).symmetric_difference(set(correct_output))
|
||||
if diff:
|
||||
log.debug("Test %s failed", case)
|
||||
log.debug("Expected:\n\t%s\nGot\n\t%s", output, correct_output)
|
||||
log.debug("Difference:\n\t%s", diff)
|
||||
self.assertEqual(output, correct_output)
|
||||
|
||||
def test_get_policies_for_nonexisting_minions(self):
|
||||
minion_id = "salt_master"
|
||||
# For non-existing minions, or the master-minion, grains will be None
|
||||
cases = {
|
||||
"no-tokens-to-replace": ["no-tokens-to-replace"],
|
||||
"single-dict:{minion}": ["single-dict:{}".format(minion_id)],
|
||||
"single-list:{grains[roles]}": [],
|
||||
"single-grain:{grains[id]}": [],
|
||||
}
|
||||
with patch(
|
||||
"salt.utils.minions.get_minion_data",
|
||||
MagicMock(return_value=(None, None, None)),
|
||||
):
|
||||
for case, correct_output in cases.items():
|
||||
test_config = {"policies": [case]}
|
||||
output = vault._get_policies(
|
||||
minion_id, test_config
|
||||
) # pylint: disable=protected-access
|
||||
diff = set(output).symmetric_difference(set(correct_output))
|
||||
if diff:
|
||||
log.debug("Test %s failed", case)
|
||||
log.debug("Expected:\n\t%s\nGot\n\t%s", output, correct_output)
|
||||
log.debug("Difference:\n\t%s", diff)
|
||||
self.assertEqual(output, correct_output)
|
||||
with patch.dict(
|
||||
vault.__utils__,
|
||||
{
|
||||
"vault.expand_pattern_lists": Mock(
|
||||
side_effect=lambda x, *args, **kwargs: [x]
|
||||
)
|
||||
},
|
||||
):
|
||||
test_config = {"policies": [case]}
|
||||
output = vault._get_policies(
|
||||
minion_id, test_config
|
||||
) # pylint: disable=protected-access
|
||||
diff = set(output).symmetric_difference(set(correct_output))
|
||||
if diff:
|
||||
log.debug("Test %s failed", case)
|
||||
log.debug("Expected:\n\t%s\nGot\n\t%s", output, correct_output)
|
||||
log.debug("Difference:\n\t%s", diff)
|
||||
self.assertEqual(output, correct_output)
|
||||
|
||||
def test_get_policies(self):
|
||||
"""
|
||||
Ensure _get_policies works as intended, including expansion of lists
|
||||
Ensure _get_policies works as intended.
|
||||
The expansion of lists is tested in the vault utility module unit tests.
|
||||
"""
|
||||
cases = {
|
||||
"no-tokens-to-replace": ["no-tokens-to-replace"],
|
||||
"single-dict:{minion}": ["single-dict:test-minion"],
|
||||
"single-list:{grains[roles]}": ["single-list:web", "single-list:database"],
|
||||
"multiple-lists:{grains[roles]}+{grains[aux]}": [
|
||||
"multiple-lists:web+foo",
|
||||
"multiple-lists:web+bar",
|
||||
"multiple-lists:database+foo",
|
||||
"multiple-lists:database+bar",
|
||||
],
|
||||
"single-list-with-dicts:{grains[id]}+{grains[roles]}+{grains[id]}": [
|
||||
"single-list-with-dicts:test-minion+web+test-minion",
|
||||
"single-list-with-dicts:test-minion+database+test-minion",
|
||||
],
|
||||
"deeply-nested-list:{grains[deep][foo][bar][baz]}": [
|
||||
"deeply-nested-list:hello",
|
||||
"deeply-nested-list:world",
|
||||
],
|
||||
"should-not-cause-an-exception,but-result-empty:{foo}": [],
|
||||
"Case-Should-Be-Lowered:{grains[mixedcase]}": [
|
||||
"case-should-be-lowered:up-low-up"
|
||||
|
|
@ -129,16 +84,24 @@ class VaultTest(TestCase, LoaderModuleMockMixin):
|
|||
MagicMock(return_value=(None, self.grains, None)),
|
||||
):
|
||||
for case, correct_output in cases.items():
|
||||
test_config = {"policies": [case]}
|
||||
output = vault._get_policies(
|
||||
"test-minion", test_config
|
||||
) # pylint: disable=protected-access
|
||||
diff = set(output).symmetric_difference(set(correct_output))
|
||||
if diff:
|
||||
log.debug("Test %s failed", case)
|
||||
log.debug("Expected:\n\t%s\nGot\n\t%s", output, correct_output)
|
||||
log.debug("Difference:\n\t%s", diff)
|
||||
self.assertEqual(output, correct_output)
|
||||
with patch.dict(
|
||||
vault.__utils__,
|
||||
{
|
||||
"vault.expand_pattern_lists": Mock(
|
||||
side_effect=lambda x, *args, **kwargs: [x]
|
||||
)
|
||||
},
|
||||
):
|
||||
test_config = {"policies": [case]}
|
||||
output = vault._get_policies(
|
||||
"test-minion", test_config
|
||||
) # pylint: disable=protected-access
|
||||
diff = set(output).symmetric_difference(set(correct_output))
|
||||
if diff:
|
||||
log.debug("Test %s failed", case)
|
||||
log.debug("Expected:\n\t%s\nGot\n\t%s", output, correct_output)
|
||||
log.debug("Difference:\n\t%s", diff)
|
||||
self.assertEqual(output, correct_output)
|
||||
|
||||
def test_get_token_create_url(self):
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue