added idem exec and state modules

changelog/57969.added

@@ -0,0 +1,2 @@
+- Added an execution module for running idem exec modules
+- Added a state module for running idem states

doc/ref/modules/all/index.rst

@@ -194,6 +194,7 @@ execution modules
     hosts
     http
     icinga2
+    idem
     ifttt
     ilo
     incron

doc/ref/modules/all/salt.modules.idem.rst

@@ -0,0 +1,5 @@
+salt.modules.idem module
+========================
+
+.. automodule:: salt.modules.idem
+    :members:

doc/ref/states/all/index.rst

@@ -131,6 +131,7 @@ state modules
     host
     http
     icinga2
+    idem
     ifttt
     incron
     influxdb08_database

doc/ref/states/all/salt.states.idem.rst

@@ -0,0 +1,5 @@
+salt.states.idem
+================
+
+.. automodule:: salt.states.idem
+    :members:

salt/modules/idem.py

@@ -0,0 +1,71 @@
+#
+# Author: Tyler Johnson <tjohnson@saltstack.com>
+#
+
+"""
+Idem Support
+============
+
+This module provides access to idem execution modules
+
+.. versionadded:: Magnesium
+"""
+# Function alias to make sure not to shadow built-in's
+__func_alias__ = {"exec_": "exec"}
+__virtualname__ = "idem"
+
+
+def __virtual__():
+    if "idem.hub" in __utils__:
+        return __virtualname__
+    else:
+        return False, "idem is not available"
+
+
+def exec_(path, acct_file=None, acct_key=None, acct_profile=None, *args, **kwargs):
+    """
+    Call an idem execution module
+
+    path
+        The idem path of the idem execution module to run
+
+    acct_file
+        Path to the acct file used in generating idem ctx parameters.
+        Defaults to the value in the ACCT_FILE environment variable.
+
+    acct_key
+        Key used to decrypt the acct file.
+        Defaults to the value in the ACCT_KEY environment variable.
+
+    acct_profile
+        Name of the profile to add to idem's ctx.acct parameter.
+        Defaults to the value in the ACCT_PROFILE environment variable.
+
+    args
+        Any positional arguments to pass to the idem exec function
+
+    kwargs
+        Any keyword arguments to pass to the idem exec function
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt '*' idem.exec test.ping
+
+    :maturity:      new
+    :depends:       acct, pop, pop-config, idem
+    :platform:      all
+    """
+    hub = __utils__["idem.hub"]()
+
+    coro = hub.idem.ex.run(
+        path,
+        args,
+        {k: v for k, v in kwargs.items() if not k.startswith("__")},
+        acct_file=acct_file or hub.OPT.acct.acct_file,
+        acct_key=acct_key or hub.OPT.acct.acct_key,
+        acct_profile=acct_profile or hub.OPT.acct.acct_profile or "default",
+    )
+
+    return hub.pop.Loop.run_until_complete(coro)

salt/states/idem.py

@@ -0,0 +1,158 @@
+#
+# Author: Tyler Johnson <tjohnson@saltstack.com>
+#
+
+"""
+Idem Support
+============
+
+This state provides access to idem states
+
+.. versionadded:: Magnesium
+"""
+import pathlib
+import re
+
+__virtualname__ = "idem"
+
+
+def __virtual__():
+    if "idem.hub" in __utils__:
+        return __virtualname__
+    else:
+        return False, "idem is not available"
+
+
+def _get_refs(sources, tree):
+    """
+    Determine where the sls sources are
+    """
+    sls_sources = []
+    SLSs = []
+    if tree:
+        sls_sources.append("file://{}".format(tree))
+    for sls in sources:
+        path = pathlib.Path(sls)
+        if path.is_file():
+            ref = str(path.stem if path.suffix == ".sls" else path.name)
+            SLSs.append(ref)
+            implied = "file://{}".format(path.parent)
+            if implied not in sls_sources:
+                sls_sources.append(implied)
+        else:
+            SLSs.append(sls)
+    return sls_sources, SLSs
+
+
+def _get_low_data(low_data):
+    """
+    Get salt-style low data from an idem state name
+    """
+    # state_|-id_|-name_|-function
+    match = re.match(r"(\w+)_\|-(\w+)\|-(\w+)_\|-(\w+)", low_data)
+    return {
+        "state": match.group(1),
+        "__id__": match.group(2),
+        "name": match.group(3),
+        "fun": match.group(4),
+    }
+
+
+def state(
+    name,
+    sls,
+    acct_file=None,
+    acct_key=None,
+    acct_profile=None,
+    cache_dir=None,
+    render=None,
+    runtime=None,
+    source_dir=None,
+    test=False,
+):
+    """
+    Execute an idem sls file through a salt state
+
+    sls
+        A list of idem sls files or sources
+
+    acct_file
+        Path to the acct file used in generating idem ctx parameters.
+        Defaults to the value in the ACCT_FILE environment variable.
+
+    acct_key
+        Key used to decrypt the acct file.
+        Defaults to the value in the ACCT_KEY environment variable.
+
+    acct_profile
+        Name of the profile to add to idem's ctx.acct parameter
+        Defaults to the value in the ACCT_PROFILE environment variable.
+
+    cache_dir
+        The location to use for the cache directory
+
+    render
+        The render pipe to use, this allows for the language to be specified (jinja|yaml)
+
+    runtime
+        Select which execution runtime to use (serial|parallel)
+
+    source_dir
+        The directory containing sls files
+
+    .. code-block:: yaml
+
+    cheese:
+      idem.state:
+        - runtime: parallel
+        - sls:
+          - idem_state.sls
+          - sls_source
+
+    :maturity:      new
+    :depends:       acct, pop, pop-config, idem
+    :platform:      all
+    """
+    hub = __utils__["idem.hub"]()
+
+    if isinstance(sls, str):
+        sls = [sls]
+
+    sls_sources, SLSs = _get_refs(sls, source_dir or hub.OPT.idem.tree)
+
+    coro = hub.idem.state.apply(
+        name=name,
+        sls_sources=sls_sources,
+        render=render or hub.OPT.idem.render,
+        runtime=runtime or hub.OPT.idem.runtime,
+        subs=["states"],
+        cache_dir=cache_dir or hub.OPT.idem.cache_dir,
+        sls=SLSs,
+        test=test,
+        acct_file=acct_file or hub.OPT.acct.acct_file,
+        acct_key=acct_key or hub.OPT.acct.acct_key,
+        acct_profile=acct_profile or hub.OPT.acct.acct_profile or "default",
+    )
+    hub.pop.Loop.run_until_complete(coro)
+
+    errors = hub.idem.RUNS[name]["errors"]
+    success = not errors
+
+    running = []
+    for idem_name, idem_return in hub.idem.RUNS[name]["running"].items():
+        standardized_idem_return = {
+            "name": idem_return["name"],
+            "changes": idem_return["changes"],
+            "result": idem_return["result"],
+            "comment": idem_return.get("comment"),
+            "low": _get_low_data(idem_name),
+        }
+        running.append(standardized_idem_return)
+
+    return {
+        "name": name,
+        "result": success,
+        "comment": "Ran {} idem states".format(len(running)) if success else errors,
+        "changes": {},
+        "sub_state_run": running,
+    }

salt/utils/idem.py

@@ -0,0 +1,57 @@
+"""
+Idem Support
+============
+
+This util provides access to an idem-ready hub
+
+.. versionadded:: Magnesium
+"""
+import logging
+import sys
+
+try:
+    import pop.hub
+
+    HAS_POP = True, None
+except ImportError as e:
+    HAS_POP = False, str(e)
+
+log = logging.getLogger(__name__)
+
+__virtualname__ = "idem"
+
+
+def __virtual__():
+    if sys.version_info < (3, 6):
+        return False, "idem only works on python3.6 and later"
+    if not HAS_POP[0]:
+        return HAS_POP
+    return __virtualname__
+
+
+def hub():
+    """
+    Create a hub with idem ready to go and completely loaded
+    """
+    if "idem.hub" not in __context__:
+        log.debug("Creating the POP hub")
+        hub = pop.hub.Hub()
+
+        log.debug("Initializing the loop")
+        hub.pop.loop.create()
+
+        log.debug("Loading subs onto hub")
+        hub.pop.sub.add(dyne_name="acct")
+        hub.pop.sub.add(dyne_name="config")
+        # We aren't collecting grains at all but some exec modules depend on the sub being on the hub
+        hub.pop.sub.add(dyne_name="grains")
+        hub.pop.sub.add(dyne_name="idem")
+        hub.pop.sub.add(dyne_name="exec")
+        hub.pop.sub.add(dyne_name="states")
+
+        log.debug("Reading idem config options")
+        hub.config.integrate.load(["acct", "idem"], "idem", parse_cli=False, logs=False)
+
+        __context__["idem.hub"] = hub
+
+    return __context__["idem.hub"]

tests/integration/modules/test_idem.py

@@ -0,0 +1,16 @@
+"""
+Integration tests for the idem execution module
+"""
+from contextlib import contextmanager
+
+import pytest
+import salt.utils.idem as idem
+import salt.utils.path
+
+
+@pytest.mark.skipif(not idem.HAS_POP[0], reason=idem.HAS_POP[1])
+@pytest.mark.skipif(not salt.utils.path.which("idem"), reason="idem is not installed")
+@contextmanager
+def test_exec(salt_call_cli):
+    ret = salt_call_cli.run("--local", "idem.exec", "test.ping")
+    assert ret.json is True

tests/integration/states/test_idem.py

@@ -0,0 +1,48 @@
+"""
+Tests for the idem state
+"""
+import tempfile
+from contextlib import contextmanager
+
+import pytest
+import salt.utils.idem as idem
+import salt.utils.path
+
+SLS_SUCCEED_WITHOUT_CHANGES = """
+state_name:
+  test.succeed_without_changes:
+    - name: idem_test
+    - foo: bar
+"""
+
+
+@pytest.mark.skipif(not idem.HAS_POP[0], reason=idem.HAS_POP[1])
+@pytest.mark.skipif(not salt.utils.path.which("idem"), reason="idem is not installed")
+@contextmanager
+def test_state(salt_call_cli):
+    with tempfile.NamedTemporaryFile(suffix=".sls", delete=True, mode="w+") as fh:
+        fh.write(SLS_SUCCEED_WITHOUT_CHANGES)
+        fh.flush()
+        ret = salt_call_cli.run(
+            "--local", "state.single", "idem.state", sls=fh.name, name="idem_test"
+        )
+
+    parent = ret.json["idem_|-idem_test_|-idem_test_|-state"]
+    assert parent["result"] is True, parent["comment"]
+    sub_state_ret = parent["sub_state_run"][0]
+    assert sub_state_ret["result"] is True
+    assert sub_state_ret["name"] == "idem_test"
+    assert "Success!" in sub_state_ret["comment"]
+
+
+def test_bad_state(salt_call_cli):
+    bad_sls = "non-existant-file.sls"
+
+    ret = salt_call_cli.run(
+        "--local", "state.single", "idem.state", sls=bad_sls, name="idem_bad_test"
+    )
+    parent = ret.json["idem_|-idem_bad_test_|-idem_bad_test_|-state"]
+
+    assert parent["result"] is False
+    assert "SLS ref {} did not resolve to a file".format(bad_sls) == parent["comment"]
+    assert not parent["sub_state_run"]

tests/integration/utils/test_idem.py

@@ -0,0 +1,49 @@
+"""
+Test utility methods that the idem module and state share
+"""
+from contextlib import contextmanager
+
+import salt.utils.idem as idem
+import salt.utils.path
+from tests.support.case import TestCase
+from tests.support.unit import skipIf
+
+HAS_IDEM = not salt.utils.path.which("idem")
+
+
+@skipIf(not idem.HAS_POP[0], str(idem.HAS_POP[1]))
+@contextmanager
+class TestIdem(TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.hub = idem.hub()
+
+    def test_loop(self):
+        assert hasattr(self.hub.pop, "Loop")
+
+    def test_subs(self):
+        for sub in ("acct", "config", "idem", "exec", "states"):
+            with self.subTest(sub=sub):
+                assert hasattr(self.hub, sub)
+
+    @skipIf(not HAS_IDEM, "idem is not installed")
+    def test_idem_ex(self):
+        assert hasattr(self.hub.idem, "ex")
+
+    @skipIf(not HAS_IDEM, "idem is not installed")
+    def test_idem_state_apply(self):
+        assert hasattr(self.hub.idem.state, "apply")
+
+    @skipIf(not HAS_IDEM, "idem is not installed")
+    def test_idem_exec(self):
+        # self.hub.exec.test.ping() causes a pylint error because of "exec" in the namespace
+        assert getattr(self.hub, "exec").test.ping()
+
+    @skipIf(not HAS_IDEM, "idem is not installed")
+    def test_idem_state(self):
+        ret = self.hub.states.test.succeed_without_changes({}, "test_state")
+        assert ret["result"] is True
+
+    def test_config(self):
+        assert self.hub.OPT.acct
+        assert self.hub.OPT.idem