Adding in new files

doc/ref/roster/all/salt.roster.dir.rst

@@ -0,0 +1,6 @@
+=================
+salt.roster.dir
+=================
+
+.. automodule:: salt.roster.dir
+    :members:

salt/roster/dir.py

@@ -0,0 +1,105 @@
+"""
+Create a salt roster out of a flat directory of files.
+
+Each filename in the directory is a minion id.
+The contents of each file is rendered using the salt renderer system.
+
+Consider the following configuration for example:
+
+config/master:
+
+    ...
+    roster: dir
+    roster_dir: config/roster.d
+    ...
+
+Where the directory config/roster.d contains two files:
+
+config/roster.d/minion-x:
+
+    host: minion-x.example.com
+    port: 22
+    sudo: true
+    user: ubuntu
+
+config/roster.d/minion-y:
+
+    host: minion-y.example.com
+    port: 22
+    sudo: true
+    user: gentoo
+
+The roster would find two minions: minion-x and minion-y, with the given host, port, sudo and user settings.
+
+The directory roster also extends the concept of roster defaults by supporting a roster_domain value in config:
+
+    ...
+    roster_domain: example.org
+    ...
+
+If that option is set, then any roster without a 'host' setting will have an implicit host of
+its minion id + '.' + the roster_domain. (The default roster_domain is the empty string,
+so you can also name the files the fully qualified name of each host. However, if you do that,
+then the fully qualified name of each host is also the minion id.)
+
+This makes it possible to avoid having to specify the hostnames when you always want them to match
+their minion id plus some domain.
+"""
+
+
+import logging
+import os
+
+import salt.loader
+import salt.template
+
+log = logging.getLogger(__name__)
+
+
+def targets(tgt, tgt_type="glob", **kwargs):
+    """
+    Return the targets from the directory of flat yaml files,
+    checks opts for location.
+    """
+    roster_dir = __opts__.get("roster_dir", "/etc/salt/roster.d")
+    # Match the targets before rendering to avoid opening files unnecessarily.
+    raw = dict.fromkeys(os.listdir(roster_dir), "")
+    log.debug("Filtering %d minions in %s", len(raw), roster_dir)
+    matched_raw = __utils__["roster_matcher.targets"](raw, tgt, tgt_type, "ipv4")
+    rendered = {
+        minion_id: _render(os.path.join(roster_dir, minion_id), **kwargs)
+        for minion_id in matched_raw
+    }
+    pruned_rendered = {id_: data for id_, data in rendered.items() if data}
+    log.debug(
+        "Matched %d minions with tgt=%s and tgt_type=%s."
+        " Discarded %d matching filenames because they had rendering errors.",
+        len(rendered),
+        tgt,
+        tgt_type,
+        len(rendered) - len(pruned_rendered),
+    )
+    return pruned_rendered
+
+
+def _render(roster_file, **kwargs):
+    """
+    Render the roster file
+    """
+    renderers = salt.loader.render(__opts__, {})
+    domain = __opts__.get("roster_domain", "")
+    try:
+        result = salt.template.compile_template(
+            roster_file,
+            renderers,
+            __opts__["renderer"],
+            __opts__["renderer_blacklist"],
+            __opts__["renderer_whitelist"],
+            mask_value="passw*",
+            **kwargs
+        )
+        result.setdefault("host", "{}.{}".format(os.path.basename(roster_file), domain))
+        return result
+    except:  # pylint: disable=W0702
+        log.warning('Unable to render roster file "%s".', roster_file, exc_info=True)
+        return {}

tests/unit/files/rosters/dir/test1_us-east-2_test_badfile

@@ -0,0 +1,6 @@
+#!jinja|yaml
+host: 127.0.0.2
+port: 22
+THIS FILE IS NOT WELL FORMED YAML
+sudo: true
+user: scoundrel

tests/unit/files/rosters/dir/test1_us-east-2_test_basic

@@ -0,0 +1,5 @@
+#!jinja|yaml
+host: 127.0.0.2
+port: 22
+sudo: true
+user: scoundrel

tests/unit/files/rosters/dir/test1_us-east-2_test_domain

@@ -0,0 +1,3 @@
+#!jinja|yaml
+port: 2222
+user: george

tests/unit/roster/test_dir.py

@@ -0,0 +1,143 @@
+"""
+Test the directory roster.
+"""
+
+
+import os
+
+import salt.config
+import salt.loader
+import salt.roster.dir as dir_
+from salt.ext import six
+from tests.support import mixins
+from tests.support.paths import TESTS_DIR
+from tests.support.runtests import RUNTIME_VARS
+from tests.support.unit import TestCase, skipIf
+
+ROSTER_DIR = os.path.join(TESTS_DIR, "unit/files/rosters/dir")
+ROSTER_DOMAIN = "test.roster.domain"
+EXPECTED = {
+    "basic": {
+        "test1_us-east-2_test_basic": {
+            "host": "127.0.0.2",
+            "port": 22,
+            "sudo": True,
+            "user": "scoundrel",
+        }
+    },
+    "domain": {
+        "test1_us-east-2_test_domain": {
+            "host": "test1_us-east-2_test_domain." + ROSTER_DOMAIN,
+            "port": 2222,
+            "user": "george",
+        }
+    },
+    "empty": {
+        "test1_us-east-2_test_empty": {
+            "host": "test1_us-east-2_test_empty." + ROSTER_DOMAIN,
+        }
+    },
+}
+
+
+class DirRosterTestCase(TestCase, mixins.LoaderModuleMockMixin):
+    """Test the directory roster"""
+
+    def setup_loader_modules(self):
+        opts = salt.config.master_config(
+            os.path.join(RUNTIME_VARS.TMP_CONF_DIR, "master")
+        )
+        utils = salt.loader.utils(
+            opts, whitelist=["json", "stringutils", "roster_matcher"]
+        )
+        runner = salt.loader.runner(opts, utils=utils, whitelist=["salt"])
+        return {
+            dir_: {
+                "__opts__": {
+                    "extension_modules": "",
+                    "optimization_order": [0, 1, 2],
+                    "renderer": "jinja|yaml",
+                    "renderer_blacklist": [],
+                    "renderer_whitelist": [],
+                    "roster_dir": ROSTER_DIR,
+                    "roster_domain": ROSTER_DOMAIN,
+                },
+                "__runner__": runner,
+                "__utils__": utils,
+            }
+        }
+
+    def _test_match(self, ret, expected):
+        """
+        assertDictEquals is too strict with OrderedDicts. The order isn't crucial
+        for roster entries, so we test that they contain the expected members directly.
+        """
+        self.assertNotEqual(ret, {}, "Found no matches, expected {}".format(expected))
+        for minion, data in ret.items():
+            self.assertIn(
+                minion,
+                expected,
+                "Expected minion {} to match, but it did not".format(minion),
+            )
+            self.assertDictEqual(
+                dict(data),
+                expected[minion],
+                "Data for minion {} did not match expectations".format(minion),
+            )
+
+    def test_basic_glob(self):
+        """Test that minion files in the directory roster match and render."""
+        expected = EXPECTED["basic"]
+        ret = dir_.targets("*_basic", saltenv="")
+        self._test_match(ret, expected)
+
+    def test_basic_re(self):
+        """Test that minion files in the directory roster match and render."""
+        expected = EXPECTED["basic"]
+        ret = dir_.targets(".*basic$", "pcre", saltenv="")
+        self._test_match(ret, expected)
+
+    def test_basic_list(self):
+        """Test that minion files in the directory roster match and render."""
+        expected = EXPECTED["basic"]
+        ret = dir_.targets(expected.keys(), "list", saltenv="")
+        self._test_match(ret, expected)
+
+    def test_roster_domain(self):
+        """Test that when roster_domain is configured, it will provide a default hostname
+        in the roster of {filename}.{roster_domain}, so that users can use the minion
+        id as the local hostname without having to supply the fqdn everywhere."""
+        expected = EXPECTED["domain"]
+        ret = dir_.targets(expected.keys(), "list", saltenv="")
+        self._test_match(ret, expected)
+
+    def test_empty(self):
+        """Test that an empty roster file matches its hostname"""
+        expected = EXPECTED["empty"]
+        ret = dir_.targets("*_empty", saltenv="")
+        self._test_match(ret, expected)
+
+    def test_nomatch(self):
+        """Test that no errors happen when no files match"""
+        try:
+            ret = dir_.targets("", saltenv="")
+        except:
+            self.fail(
+                "No files matched, which is OK, but we raised an exception and we should not have."
+            )
+            raise
+        self.assertEqual(
+            len(ret), 0, "Expected empty target list to yield zero targets."
+        )
+
+    def test_badfile(self):
+        """Test error handling when we can't render a file"""
+        ret = dir_.targets("*badfile", saltenv="")
+        self.assertEqual(len(ret), 0)
+
+    @skipIf(not six.PY3, "Can only assertLogs in PY3")
+    def test_badfile_logging(self):
+        """Test error handling when we can't render a file"""
+        with self.assertLogs("salt.roster.dir", level="WARNING") as logged:
+            dir_.targets("*badfile", saltenv="")
+            self.assertIn("test1_us-east-2_test_badfile", logged.output[0])