Allow accessing the regular mine/event bus from salt-ssh

changelog/40943.added.md

@@ -0,0 +1 @@
+Allowed publishing to regular minions from the SSH wrapper

changelog/65645.added.md

@@ -0,0 +1 @@
+Allowed accessing the regular mine from the SSH wrapper

salt/client/ssh/wrapper/mine.py

@@ -2,26 +2,35 @@
 Wrapper function for mine operations for salt-ssh
 
 .. versionadded:: 2015.5.0
+.. versionchanged:: 3007.0
+
+    In addition to mine returns from roster targets, this wrapper now supports
+    accessing the regular mine as well.
 """
 
 
 import copy
 
 import salt.client.ssh
+import salt.daemons.masterapi
 
 
-def get(tgt, fun, tgt_type="glob", roster="flat"):
+def get(
+    tgt, fun, tgt_type="glob", roster="flat", ssh_minions=True, regular_minions=False
+):
     """
     Get data from the mine based on the target, function and tgt_type
 
-    This will actually run the function on all targeted minions (like
+    This will actually run the function on all targeted SSH minions (like
     publish.publish), as salt-ssh clients can't update the mine themselves.
 
     We will look for mine_functions in the roster, pillar, and master config,
-    in that order, looking for a match for the defined function
+    in that order, looking for a match for the defined function.
 
     Targets can be matched based on any standard matching system that can be
-    matched on the defined roster (in salt-ssh) via these keywords::
+    matched on the defined roster (in salt-ssh).
+
+    Regular mine data will be fetched as usual and can be targeted as usual.
 
     CLI Example:
 
@@ -30,30 +39,67 @@ def get(tgt, fun, tgt_type="glob", roster="flat"):
         salt-ssh '*' mine.get '*' network.interfaces
         salt-ssh '*' mine.get 'myminion' network.interfaces roster=flat
         salt-ssh '*' mine.get '192.168.5.0' network.ipaddrs roster=scan
+        salt-ssh myminion mine.get '*' network.interfaces ssh_minions=False regular_minions=True
+        salt-ssh myminion mine.get '*' network.interfaces ssh_minions=True regular_minions=True
+
+    tgt
+        Target whose mine data to get.
+
+    fun
+        Function to get the mine data of. You can specify multiple functions
+        to retrieve using either a list or a comma-separated string of functions.
+
+    tgt_type
+        Target type to use with ``tgt``. Defaults to ``glob``.
+        See :ref:`targeting` for more information for regular minion targets, above
+        for SSH ones.
+
+    roster
+        The roster module to use. Defaults to ``flat``.
+
+    ssh_minions
+        .. versionadded:: 3007.0
+        Target minions from the roster. Defaults to true.
+
+    regular_minions
+        .. versionadded:: 3007.0
+        Target regular minions of the master running salt-ssh. Defaults to false.
     """
-    # Set up opts for the SSH object
-    opts = copy.deepcopy(__context__["master_opts"])
-    minopts = copy.deepcopy(__opts__)
-    opts.update(minopts)
-    if roster:
-        opts["roster"] = roster
-    opts["argv"] = [fun]
-    opts["selected_target_option"] = tgt_type
-    opts["tgt"] = tgt
-    opts["arg"] = []
-
-    # Create the SSH object to handle the actual call
-    ssh = salt.client.ssh.SSH(opts)
-
-    # Run salt-ssh to get the minion returns
     rets = {}
-    for ret in ssh.run_iter(mine=True):
+    if regular_minions:
+        masterapi = salt.daemons.masterapi.RemoteFuncs(__context__["master_opts"])
+        load = {
+            "id": __opts__["id"],
+            "fun": fun,
+            "tgt": tgt,
+            "tgt_type": tgt_type,
+        }
+        ret = masterapi._mine_get(load)
         rets.update(ret)
 
-    cret = {}
-    for host in rets:
-        if "return" in rets[host]:
-            cret[host] = rets[host]["return"]
-        else:
-            cret[host] = rets[host]
-    return cret
+    if ssh_minions:
+        # Set up opts for the SSH object
+        opts = copy.deepcopy(__context__["master_opts"])
+        minopts = copy.deepcopy(__opts__)
+        opts.update(minopts)
+        if roster:
+            opts["roster"] = roster
+        opts["argv"] = [fun]
+        opts["selected_target_option"] = tgt_type
+        opts["tgt"] = tgt
+        opts["arg"] = []
+
+        # Create the SSH object to handle the actual call
+        ssh = salt.client.ssh.SSH(opts)
+
+        # Run salt-ssh to get the minion returns
+        mrets = {}
+        for ret in ssh.run_iter(mine=True):
+            mrets.update(ret)
+
+        for host in mrets:
+            if "return" in mrets[host]:
+                rets[host] = mrets[host]["return"]
+            else:
+                rets[host] = mrets[host]
+    return rets

salt/client/ssh/wrapper/publish.py

@@ -7,18 +7,39 @@ Publish will never actually execute on the minions, so we just create new
 salt-ssh calls and return the data from them.
 
 No access control is needed because calls cannot originate from the minions.
+
+.. versionchanged:: 3007.0
+
+    In addition to SSH minions, this module can now also target regular ones.
 """
 
 import copy
 import logging
+import time
 
 import salt.client.ssh
+import salt.daemons.masterapi
 import salt.runner
 import salt.utils.args
+import salt.utils.json
 
 log = logging.getLogger(__name__)
 
 
+def _parse_args(arg):
+    """
+    yamlify `arg` and ensure its outermost datatype is a list
+    """
+    yaml_args = salt.utils.args.yamlify_arg(arg)
+
+    if yaml_args is None:
+        return []
+    elif not isinstance(yaml_args, list):
+        return [yaml_args]
+    else:
+        return yaml_args
+
+
 def _publish(
     tgt,
     fun,
@@ -65,9 +86,13 @@ def _publish(
     if arg is None:
         arg = []
     elif not isinstance(arg, list):
-        arg = [salt.utils.args.yamlify_arg(arg)]
+        # yamlify_arg does not operate on non-strings, which we need to JSON-encode
+        arg = [salt.utils.json.dumps(salt.utils.args.yamlify_arg(arg))]
     else:
-        arg = [salt.utils.args.yamlify_arg(x) for x in arg]
+        arg = [
+            salt.utils.json.dumps(y)
+            for y in (salt.utils.args.yamlify_arg(x) for x in arg)
+        ]
     if len(arg) == 1 and arg[0] is None:
         arg = []
 
@@ -100,29 +125,130 @@ def _publish(
             else:
                 cret[host] = rets[host]
         return cret
+    for host in rets:
+        if "return" in rets[host]:
+            # The regular publish return just contains `ret`,
+            # at least make it accessible like this as well
+            rets[host]["ret"] = rets[host]["return"]
+    return rets
+
+
+def _publish_regular(
+    tgt,
+    fun,
+    arg=None,
+    tgt_type="glob",
+    returner="",
+    timeout=5,
+    form="clean",
+    wait=False,
+):
+    if fun.startswith("publish."):
+        log.info("Cannot publish publish calls. Returning {}")
+        return {}
+
+    arg = _parse_args(arg)
+    masterapi = salt.daemons.masterapi.RemoteFuncs(__context__["master_opts"])
+
+    log.info("Publishing '%s'", fun)
+    load = {
+        "cmd": "minion_pub",
+        "fun": fun,
+        "arg": arg,
+        "tgt": tgt,
+        "tgt_type": tgt_type,
+        "ret": returner,
+        "tmo": timeout,
+        "form": form,
+        "id": __opts__["id"],
+        "no_parse": __opts__.get("no_parse", []),
+    }
+    peer_data = masterapi.minion_pub(load)
+    if not peer_data:
+        return {}
+    # CLI args are passed as strings, re-cast to keep time.sleep happy
+    if wait:
+        loop_interval = 0.3
+        matched_minions = set(peer_data["minions"])
+        returned_minions = set()
+        loop_counter = 0
+        while returned_minions ^ matched_minions:
+            load = {
+                "cmd": "pub_ret",
+                "id": __opts__["id"],
+                "jid": peer_data["jid"],
+            }
+            ret = masterapi.pub_ret(load)
+            returned_minions = set(ret.keys())
+
+            end_loop = False
+            if returned_minions >= matched_minions:
+                end_loop = True
+            elif (loop_interval * loop_counter) > timeout:
+                if not returned_minions:
+                    return {}
+                end_loop = True
+
+            if end_loop:
+                if form == "clean":
+                    cret = {}
+                    for host in ret:
+                        cret[host] = ret[host]["ret"]
+                    return cret
+                else:
+                    return ret
+            loop_counter = loop_counter + 1
+            time.sleep(loop_interval)
     else:
-        return rets
+        time.sleep(float(timeout))
+        load = {
+            "cmd": "pub_ret",
+            "id": __opts__["id"],
+            "jid": peer_data["jid"],
+        }
+        ret = masterapi.pub_ret(load)
+        if form == "clean":
+            cret = {}
+            for host in ret:
+                cret[host] = ret[host]["ret"]
+            return cret
+        else:
+            return ret
+    return ret
 
 
-def publish(tgt, fun, arg=None, tgt_type="glob", returner="", timeout=5, roster=None):
+def publish(
+    tgt,
+    fun,
+    arg=None,
+    tgt_type="glob",
+    returner="",
+    timeout=5,
+    roster=None,
+    ssh_minions=True,
+    regular_minions=False,
+):
     """
-    Publish a command "from the minion out to other minions". In reality, the
+    Publish a command from the minion out to other minions. In reality, the
     minion does not execute this function, it is executed by the master. Thus,
     no access control is enabled, as minions cannot initiate publishes
     themselves.
 
-
     Salt-ssh publishes will default to whichever roster was used for the
     initiating salt-ssh call, and can be overridden using the ``roster``
-    argument
+    argument.
 
     Returners are not currently supported
 
     The tgt_type argument is used to pass a target other than a glob into
-    the execution, the available options are:
+    the execution, the available options for SSH minions are:
 
     - glob
     - pcre
+    - nodegroup
+    - range
+
+    Regular minions support all usual ones.
 
     .. versionchanged:: 2017.7.0
         The ``expr_form`` argument has been renamed to ``tgt_type``, earlier
@@ -161,21 +287,77 @@ def publish(tgt, fun, arg=None, tgt_type="glob", returner="", timeout=5, roster=
             salt-ssh '*' publish.publish test.kwarg arg="['cheese=spam','spam=cheese']"
 
 
+    tgt
+        The target specification.
 
+    fun
+        The execution module to run.
+
+    arg
+        A list of arguments to pass to the module.
+
+    tgt_type
+        The matcher to use. Defaults to ``glob``.
+
+    returner
+        A returner to use.
+
+    timeout
+        Timeout in seconds. Defaults to 5.
+
+    roster
+        Override the roster for SSH minion targets. Defaults to the one
+        used for initiating the salt-ssh call.
+
+    ssh_minions
+        .. versionadded:: 3007.0
+        Include SSH minions in the possible targets. Defaults to true.
+
+    regular_minions
+        .. versionadded:: 3007.0
+        Include regular minions in the possible targets. Defaults to false.
     """
-    return _publish(
-        tgt,
-        fun,
-        arg=arg,
-        tgt_type=tgt_type,
-        returner=returner,
-        timeout=timeout,
-        form="clean",
-        roster=roster,
-    )
+    rets = {}
+    if regular_minions:
+        rets.update(
+            _publish_regular(
+                tgt,
+                fun,
+                arg=arg,
+                tgt_type=tgt_type,
+                returner=returner,
+                timeout=timeout,
+                form="clean",
+                wait=True,
+            )
+        )
+    if ssh_minions:
+        rets.update(
+            _publish(
+                tgt,
+                fun,
+                arg=arg,
+                tgt_type=tgt_type,
+                returner=returner,
+                timeout=timeout,
+                form="clean",
+                roster=roster,
+            )
+        )
+    return rets
 
 
-def full_data(tgt, fun, arg=None, tgt_type="glob", returner="", timeout=5, roster=None):
+def full_data(
+    tgt,
+    fun,
+    arg=None,
+    tgt_type="glob",
+    returner="",
+    timeout=5,
+    roster=None,
+    ssh_minions=True,
+    regular_minions=False,
+):
     """
     Return the full data about the publication, this is invoked in the same
     way as the publish function
@@ -197,21 +379,39 @@ def full_data(tgt, fun, arg=None, tgt_type="glob", returner="", timeout=5, roste
             salt-ssh '*' publish.full_data test.kwarg arg='cheese=spam'
 
     """
-    return _publish(
-        tgt,
-        fun,
-        arg=arg,
-        tgt_type=tgt_type,
-        returner=returner,
-        timeout=timeout,
-        form="full",
-        roster=roster,
-    )
+    rets = {}
+    if regular_minions:
+        rets.update(
+            _publish_regular(
+                tgt,
+                fun,
+                arg=arg,
+                tgt_type=tgt_type,
+                returner=returner,
+                timeout=timeout,
+                form="full",
+                wait=True,
+            )
+        )
+    if ssh_minions:
+        rets.update(
+            _publish(
+                tgt,
+                fun,
+                arg=arg,
+                tgt_type=tgt_type,
+                returner=returner,
+                timeout=timeout,
+                form="full",
+                roster=roster,
+            )
+        )
+    return rets
 
 
 def runner(fun, arg=None, timeout=5):
     """
-    Execute a runner on the master and return the data from the runnr function
+    Execute a runner on the master and return the data from the runner function
 
     CLI Example:
 

tests/pytests/integration/ssh/test_mine.py

@@ -29,3 +29,28 @@ def test_ssh_mine_get(salt_ssh_cli):
     assert "localhost" in ret.data
     assert "args" in ret.data["localhost"]
     assert ret.data["localhost"]["args"] == ["itworked"]
+
+
+@pytest.mark.parametrize("tgts", (("ssh",), ("regular",), ("ssh", "regular")))
+def test_mine_get(salt_ssh_cli, salt_minion, tgts):
+    """
+    Test mine returns with both regular and SSH minions
+    """
+    if len(tgts) > 1:
+        tgt = "*"
+        exp = {"localhost", salt_minion.id}
+    else:
+        tgt = "localhost" if "ssh" in tgts else salt_minion.id
+        exp = {tgt}
+    ret = salt_ssh_cli.run(
+        "mine.get",
+        "*",
+        "test.ping",
+        ssh_minions="ssh" in tgts,
+        regular_minions="regular" in tgts,
+    )
+    assert ret.returncode == 0
+    assert ret.data
+    assert set(ret.data) == exp
+    for id_ in exp:
+        assert ret.data[id_] is True

tests/pytests/integration/ssh/test_publish.py

@@ -0,0 +1,106 @@
+import pytest
+
+pytestmark = [
+    pytest.mark.slow_test,
+    pytest.mark.skip_on_windows(reason="salt-ssh not available on Windows"),
+]
+
+
+@pytest.mark.parametrize("tgts", (("ssh",), ("regular",), ("ssh", "regular")))
+def test_publish(salt_ssh_cli, salt_minion, tgts):
+    if len(tgts) > 1:
+        tgt = "*"
+        exp = {"localhost", salt_minion.id}
+    else:
+        tgt = "localhost" if "ssh" in tgts else salt_minion.id
+        exp = {tgt}
+    ret = salt_ssh_cli.run(
+        "publish.publish",
+        tgt,
+        "test.ping",
+        ssh_minions="ssh" in tgts,
+        regular_minions="regular" in tgts,
+    )
+    assert ret.returncode == 0
+    assert ret.data
+    assert set(ret.data) == exp
+    for id_ in exp:
+        assert ret.data[id_] is True
+
+
+def test_publish_with_arg(salt_ssh_cli, salt_minion):
+    ret = salt_ssh_cli.run(
+        "publish.publish",
+        "*",
+        "test.kwarg",
+        arg=["cheese=spam"],
+        ssh_minions=True,
+        regular_minions=True,
+    )
+    assert ret.returncode == 0
+    assert ret.data
+    exp = {salt_minion.id, "localhost"}
+    assert set(ret.data) == exp
+    for id_ in exp:
+        assert ret.data[id_]["cheese"] == "spam"
+
+
+def test_publish_with_yaml_args(salt_ssh_cli, salt_minion):
+    args = ["saltines, si", "crackers, nein", "cheese, indeed"]
+    test_args = f'["{args[0]}", "{args[1]}", "{args[2]}"]'
+    ret = salt_ssh_cli.run(
+        "publish.publish",
+        "*",
+        "test.arg",
+        arg=test_args,
+        ssh_minions=True,
+        regular_minions=True,
+    )
+    assert ret.returncode == 0
+    assert ret.data
+    exp = {salt_minion.id, "localhost"}
+    assert set(ret.data) == exp
+    for id_ in exp:
+        assert ret.data[id_]["args"] == args
+
+
+@pytest.mark.parametrize("tgts", (("ssh",), ("regular",), ("ssh", "regular")))
+def test_full_data(salt_ssh_cli, salt_minion, tgts):
+    if len(tgts) > 1:
+        tgt = "*"
+        exp = {"localhost", salt_minion.id}
+    else:
+        tgt = "localhost" if "ssh" in tgts else salt_minion.id
+        exp = {tgt}
+    ret = salt_ssh_cli.run(
+        "publish.full_data",
+        tgt,
+        "test.fib",
+        arg=20,
+        ssh_minions="ssh" in tgts,
+        regular_minions="regular" in tgts,
+    )
+    assert ret.returncode == 0
+    assert ret.data
+    assert set(ret.data) == exp
+    for id_ in exp:
+        assert "ret" in ret.data[id_]
+        assert ret.data[id_]["ret"][0] == 6765
+
+
+def test_full_data_kwarg(salt_ssh_cli, salt_minion):
+    ret = salt_ssh_cli.run(
+        "publish.full_data",
+        "*",
+        "test.kwarg",
+        arg=["cheese=spam"],
+        ssh_minions=True,
+        regular_minions=True,
+    )
+    assert ret.returncode == 0
+    assert ret.data
+    exp = {"localhost", salt_minion.id}
+    assert set(ret.data) == exp
+    for id_ in exp:
+        assert "ret" in ret.data[id_]
+        assert ret.data[id_]["ret"]["cheese"] == "spam"