Merge pull request #56685 from mchugh19/baredoc_ast

Module to list arguments to salt modules without loading

doc/ref/modules/all/index.rst

@@ -47,6 +47,7 @@ execution modules
     azurearm_network
     azurearm_resource
     bamboohr
+    baredoc
     bcache
     beacons
     bigip

doc/ref/modules/all/salt.modules.baredoc.rst

@@ -0,0 +1,5 @@
+salt.modules.baredoc module
+===========================
+
+.. automodule:: salt.modules.baredoc
+    :members:

salt/modules/baredoc.py

@@ -0,0 +1,250 @@
+# -*- coding: utf-8 -*-
+"""
+Baredoc walks the installed module and state directories and generates
+dictionaries and lists of the function names and their arguments.
+
+.. versionadded:: Sodium
+
+"""
+from __future__ import absolute_import, print_function, unicode_literals
+
+import ast
+
+# Import python libs
+import logging
+import os
+
+# Import salt libs
+import salt.utils.files
+from salt.ext.six.moves import zip_longest
+
+# Import 3rd-party libs
+from salt.utils.odict import OrderedDict
+
+log = logging.getLogger(__name__)
+
+
+def _get_module_name(tree, filename):
+    """
+    Returns the value of __virtual__ if found.
+    Otherwise, returns filename
+    """
+    module_name = os.path.basename(filename).split(".")[0]
+    assignments = [node for node in tree.body if isinstance(node, ast.Assign)]
+    for assign in assignments:
+        try:
+            if assign.targets[0].id == "__virtualname__":
+                module_name = assign.value.s
+        except AttributeError:
+            pass
+    return module_name
+
+
+def _get_func_aliases(tree):
+    """
+    Get __func_alias__ dict for mapping function names
+    """
+    fun_aliases = {}
+    assignments = [node for node in tree.body if isinstance(node, ast.Assign)]
+    for assign in assignments:
+        try:
+            if assign.targets[0].id == "__func_alias__":
+                for key, value in zip_longest(assign.value.keys, assign.value.values):
+                    fun_aliases.update({key.s: value.s})
+        except AttributeError:
+            pass
+    return fun_aliases
+
+
+def _get_args(function):
+    """
+    Given a function def, returns arguments and defaults
+    """
+    # Generate list of arguments
+    arg_strings = []
+    list_of_arguments = function.args.args
+    if list_of_arguments:
+        for arg in list_of_arguments:
+            arg_strings.append(arg.arg)
+
+    # Generate list of arg defaults
+    # Values are only returned for populated items
+    arg_default_strings = []
+    list_arg_defaults = function.args.defaults
+    if list_arg_defaults:
+        for arg_default in list_arg_defaults:
+            if isinstance(arg_default, ast.NameConstant):
+                arg_default_strings.append(arg_default.value)
+            elif isinstance(arg_default, ast.Str):
+                arg_default_strings.append(arg_default.s)
+            elif isinstance(arg_default, ast.Num):
+                arg_default_strings.append(arg_default.n)
+
+    # Since only some args may have default values, need to zip in reverse order
+    backwards_args = OrderedDict(
+        zip_longest(reversed(arg_strings), reversed(arg_default_strings))
+    )
+    ordered_args = OrderedDict(reversed(list(backwards_args.items())))
+
+    try:
+        ordered_args["args"] = function.args.vararg.arg
+    except AttributeError:
+        pass
+    try:
+        ordered_args["kwargs"] = function.args.kwarg.arg
+    except AttributeError:
+        pass
+
+    return ordered_args
+
+
+def _mods_with_args(module_py, names_only):
+    """
+    Start ast parsing of modules
+    """
+    ret = {}
+    with salt.utils.files.fopen(module_py, "r", encoding="utf8") as cur_file:
+        tree = ast.parse(cur_file.read())
+        module_name = _get_module_name(tree, module_py)
+        fun_aliases = _get_func_aliases(tree)
+
+        functions = [node for node in tree.body if isinstance(node, ast.FunctionDef)]
+        func_list = []
+        for fn in functions:
+            if not fn.name.startswith("_"):
+                function_name = fn.name
+                if fun_aliases:
+                    # Translate name to __func_alias__ version
+                    for k, v in fun_aliases.items():
+                        if fn.name == k:
+                            function_name = v
+                args = _get_args(fn)
+                if names_only:
+                    func_list.append(function_name)
+                else:
+                    fun_entry = {}
+                    fun_entry[function_name] = args
+                    func_list.append(fun_entry)
+        ret[module_name] = func_list
+    return ret
+
+
+def _modules_and_args(name=False, type="states", names_only=False):
+    """
+    Determine if modules/states directories or files are requested
+    """
+    ret = {}
+    dirs = []
+
+    if type == "modules":
+        dirs.append(os.path.join(__opts__["extension_modules"], "modules"))
+        dirs.append(os.path.join(__grains__["saltpath"], "modules"))
+    elif type == "states":
+        dirs.append(os.path.join(__opts__["extension_modules"], "states"))
+        dirs.append(os.path.join(__grains__["saltpath"], "states"))
+
+    if name:
+        for dir in dirs:
+            # Process custom dirs first so custom results are returned
+            if os.path.exists(os.path.join(dir, name + ".py")):
+                return _mods_with_args(os.path.join(dir, name + ".py"), names_only)
+    else:
+        for dir in reversed(dirs):
+            # Process custom dirs last so they are displayed
+            try:
+                for module_py in os.listdir(dir):
+                    if module_py.endswith(".py") and module_py != "__init__.py":
+                        ret.update(
+                            _mods_with_args(os.path.join(dir, module_py), names_only)
+                        )
+            except FileNotFoundError:
+                pass
+    return ret
+
+
+def list_states(name=False, names_only=False):
+    """
+    Walk the Salt install tree for state modules and return a
+    dictionary or a list of their functions as well as their arguments.
+
+    :param name: specify a specific module to list. If not specified, all modules will be listed.
+    :param names_only: Return only a list of the callable functions instead of a dictionary with arguments
+
+    CLI Example:
+    (example truncated for brevity)
+
+    .. code-block:: bash
+
+        salt myminion baredoc.modules_and_args
+
+        myminion:
+            ----------
+        [...]
+          at:
+          - present:
+              name: null
+              timespec: null
+              tag: null
+              user: null
+              job: null
+              unique_tag: false
+           - absent:
+              name: null
+              jobid: null
+              kwargs: kwargs
+           - watch:
+              name: null
+              timespec: null
+              tag: null
+              user: null
+              job: null
+              unique_tag: false
+           - mod_watch:
+              name: null
+              kwargs: kwargs
+        [...]
+    """
+    ret = _modules_and_args(name, type="states", names_only=names_only)
+    if names_only:
+        return OrderedDict(sorted(ret.items()))
+    else:
+        return OrderedDict(sorted(ret.items()))
+
+
+def list_modules(name=False, names_only=False):
+    """
+    Walk the Salt install tree for execution modules and return a
+    dictionary or a list of their functions as well as their arguments.
+
+    :param name: specify a specific module to list. If not specified, all modules will be listed.
+    :param names_only: Return only a list of the callable functions instead of a dictionary with arguments
+
+    CLI Example:
+    (example truncated for brevity)
+
+    .. code-block:: bash
+
+        salt myminion baredoc.modules_and_args
+
+        myminion:
+            ----------
+        [...]
+          at:
+        - atq:
+            tag: null
+          - atrm:
+            args: args
+          - at:
+            args: args
+            kwargs: kwargs
+          - atc:
+            jobid: null
+          - jobcheck:
+            kwargs: kwargs
+        [...]
+    """
+    ret = _modules_and_args(name, type="modules", names_only=names_only)
+    if names_only:
+        return OrderedDict(sorted(ret.items()))
+    else:
+        return OrderedDict(sorted(ret.items()))

tests/unit/modules/test_baredoc.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+
+# Import Python libs
+from __future__ import absolute_import
+
+import os
+
+# Import module
+import salt.modules.baredoc as baredoc
+from tests.support.mixins import LoaderModuleMockMixin
+from tests.support.runtests import RUNTIME_VARS
+
+# Import Salt Testing Libs
+from tests.support.unit import TestCase
+
+
+class BaredocTest(TestCase, LoaderModuleMockMixin):
+    """
+    Validate baredoc module
+    """
+
+    def setup_loader_modules(self):
+        return {
+            baredoc: {
+                "__opts__": {
+                    "extension_modules": os.path.join(RUNTIME_VARS.CODE_DIR, "salt"),
+                },
+                "__grains__": {"saltpath": os.path.join(RUNTIME_VARS.CODE_DIR, "salt")},
+            }
+        }
+
+    def test_baredoc_list_states(self):
+        """
+        Test baredoc state module listing
+        """
+        ret = baredoc.list_states(names_only=True)
+        assert "value_present" in ret["xml"][0]
+
+    def test_baredoc_list_states_args(self):
+        """
+        Test baredoc state listing with args
+        """
+        ret = baredoc.list_states()
+        assert "value_present" in ret["xml"][0]
+        assert "xpath" in ret["xml"][0]["value_present"]
+
+    def test_baredoc_list_states_single(self):
+        """
+        Test baredoc state listing single state module
+        """
+        ret = baredoc.list_states("xml")
+        assert "value_present" in ret["xml"][0]
+        assert "xpath" in ret["xml"][0]["value_present"]
+
+    def test_baredoc_list_modules(self):
+        """
+        test baredoc executiion module listing
+        """
+        ret = baredoc.list_modules(names_only=True)
+        assert "get_value" in ret["xml"][0]
+
+    def test_baredoc_list_modules_args(self):
+        """
+        test baredoc execution module listing with args
+        """
+        ret = baredoc.list_modules()
+        assert "get_value" in ret["xml"][0]
+        assert "file" in ret["xml"][0]["get_value"]
+
+    def test_baredoc_list_modules_single_and_alias(self):
+        """
+        test baredoc single module listing
+        """
+        ret = baredoc.list_modules("mdata")
+        assert "put" in ret["mdata"][2]
+        assert "keyname" in ret["mdata"][2]["put"]