Merge pull request #56662 from terminalmage/bp-49066

Port #49066 to master branch

doc/ref/returners/all/index.rst

@@ -10,6 +10,7 @@ returner modules
     :toctree:
     :template: autosummary.rst.tmpl
 
+    appoptics_return
     carbon_return
     cassandra_cql_return
     cassandra_return

doc/ref/returners/all/salt.returners.appoptics_return.rst

@@ -0,0 +1,6 @@
+salt.returners.appoptics_return module
+======================================
+
+.. automodule:: salt.returners.appoptics_return
+    :members:
+    :undoc-members:

salt/returners/appoptics_return.py

@@ -0,0 +1,225 @@
+# -*- coding: utf-8 -*-
+"""Salt returner to return highstate stats to AppOptics Metrics
+
+To enable this returner the minion will need the AppOptics Metrics
+client importable on the Python path and the following
+values configured in the minion or master config.
+
+The AppOptics python client can be found at:
+
+https://github.com/appoptics/python-appoptics-metrics
+
+.. code-block:: yaml
+
+    appoptics.api_token: abc12345def
+
+An example configuration that returns the total number of successes
+and failures for your salt highstate runs (the default) would look
+like this:
+
+.. code-block:: yaml
+
+    return: appoptics
+    appoptics.api_token: <token string here>
+
+
+The returner publishes the following metrics to AppOptics:
+
+- saltstack.failed
+- saltstack.passed
+- saltstack.retcode
+- saltstack.runtime
+- saltstack.total
+
+
+You can add a tags section to specify which tags should be attached to
+all metrics created by the returner.
+
+.. code-block:: yaml
+
+    appoptics.tags:
+      host_hostname_alias: <the minion ID - matches @host>
+      tier: <the tier/etc. of this node>
+      cluster: <the cluster name, etc.>
+
+
+If no tags are explicitly configured, then the tag key ``host_hostname_alias``
+will be set, with the minion's ``id`` grain being the value.
+
+In addition to the requested tags, for a highstate run each of these
+will be tagged with the ``key:value`` of ``state_type: highstate``.
+
+In order to return metrics for ``state.sls`` runs (distinct from highstates), you can
+specify a list of state names to the key ``appoptics.sls_states`` like so:
+
+.. code-block:: yaml
+
+    appoptics.sls_states:
+      - role_salt_master.netapi
+      - role_redis.config
+      - role_smarty.dummy
+
+
+This will report success and failure counts on runs of the
+``role_salt_master.netapi``, ``role_redis.config``, and
+``role_smarty.dummy`` states in addition to highstates.
+
+This will report the same metrics as above, but for these runs the
+metrics will be tagged with ``state_type: sls`` and ``state_name`` set to
+the name of the state that was invoked, e.g. ``role_salt_master.netapi``.
+
+"""
+
+# Import python libs
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+
+import salt.returners
+
+# Import Salt libs
+import salt.utils.jid
+
+# Import third party libs
+try:
+    import appoptics_metrics
+
+    HAS_APPOPTICS = True
+except ImportError:
+    HAS_APPOPTICS = False
+
+# Define the module's Virtual Name
+__virtualname__ = "appoptics"
+
+log = logging.getLogger(__name__)
+
+
+def __virtual__():
+    if not HAS_APPOPTICS:
+        log.error(
+            "The appoptics_return module couldn't load the appoptics_metrics module."
+        )
+        log.error("please make sure it is installed and is in the PYTHON_PATH.")
+        return (
+            False,
+            "Could not import appoptics_metrics module; "
+            "appoptics-metrics python client is not installed.",
+        )
+    return __virtualname__
+
+
+def _get_options(ret=None):
+    """
+    Get the appoptics options from salt.
+    """
+    attrs = {
+        "api_token": "api_token",
+        "api_url": "api_url",
+        "tags": "tags",
+        "sls_states": "sls_states",
+    }
+
+    _options = salt.returners.get_returner_options(
+        __virtualname__, ret, attrs, __salt__=__salt__, __opts__=__opts__
+    )
+
+    _options["api_url"] = _options.get("api_url", "api.appoptics.com")
+    _options["sls_states"] = _options.get("sls_states", [])
+    _options["tags"] = _options.get(
+        "tags", {"host_hostname_alias": __salt__["grains.get"]("id")}
+    )
+
+    log.debug("Retrieved appoptics options: {}".format(_options))
+    return _options
+
+
+def _get_appoptics(options):
+    """
+    Return an appoptics connection object.
+    """
+    conn = appoptics_metrics.connect(
+        options.get("api_token"),
+        sanitizer=appoptics_metrics.sanitize_metric_name,
+        hostname=options.get("api_url"),
+    )
+    log.info("Connected to appoptics.")
+    return conn
+
+
+def _calculate_runtimes(states):
+    results = {"runtime": 0.00, "num_failed_states": 0, "num_passed_states": 0}
+
+    for state, resultset in states.items():
+        if isinstance(resultset, dict) and "duration" in resultset:
+            # Count the pass vs failures
+            if resultset["result"]:
+                results["num_passed_states"] += 1
+            else:
+                results["num_failed_states"] += 1
+
+            # Count durations
+            results["runtime"] += resultset["duration"]
+
+    log.debug("Parsed state metrics: {}".format(results))
+    return results
+
+
+def _state_metrics(ret, options, tags):
+    # Calculate the runtimes and number of failed states.
+    stats = _calculate_runtimes(ret["return"])
+    log.debug("Batching Metric retcode with {}".format(ret["retcode"]))
+    appoptics_conn = _get_appoptics(options)
+    q = appoptics_conn.new_queue(tags=tags)
+
+    q.add("saltstack.retcode", ret["retcode"])
+    log.debug(
+        "Batching Metric num_failed_jobs with {}".format(stats["num_failed_states"])
+    )
+    q.add("saltstack.failed", stats["num_failed_states"])
+
+    log.debug(
+        "Batching Metric num_passed_states with {}".format(stats["num_passed_states"])
+    )
+    q.add("saltstack.passed", stats["num_passed_states"])
+
+    log.debug("Batching Metric runtime with {}".format(stats["runtime"]))
+    q.add("saltstack.runtime", stats["runtime"])
+
+    log.debug(
+        "Batching with  Metric total states {}".format(
+            (stats["num_failed_states"] + stats["num_passed_states"])
+        )
+    )
+    q.add(
+        "saltstack.highstate.total_states",
+        (stats["num_failed_states"] + stats["num_passed_states"]),
+    )
+    log.info("Sending metrics to appoptics.")
+    q.submit()
+
+
+def returner(ret):
+    """
+    Parse the return data and return metrics to AppOptics.
+
+    For each state that's provided in the configuration, return tagged metrics for
+    the result of that state if it's present.
+    """
+
+    options = _get_options(ret)
+    states_to_report = ["state.highstate"]
+    if options.get("sls_states"):
+        states_to_report.append("state.sls")
+    if ret["fun"] in states_to_report:
+        tags = options.get("tags", {}).copy()
+        tags["state_type"] = ret["fun"]
+        log.info("Tags for this run are {}".format(str(tags)))
+        matched_states = set(ret["fun_args"]).intersection(
+            set(options.get("sls_states", []))
+        )
+        # What can I do if a run has multiple states that match?
+        # In the mean time, find one matching state name and use it.
+        if matched_states:
+            tags["state_name"] = sorted(matched_states)[0]
+            log.debug("Found returned data from {}.".format(tags["state_name"]))
+        _state_metrics(ret, options, tags)

tests/integration/returners/test_appoptics_return.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+"""
+Tests for the appoptics returner
+"""
+# Import Python libs
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+
+# Import salt libs
+from salt.returners import appoptics_return
+
+# Import Salt Testing libs
+from tests.support.case import ShellCase
+
+log = logging.getLogger(__name__)
+
+# JOBS DIR and FILES
+MOCK_RET_HIGHSTATE = {
+    "fun_args": [],
+    "return": {
+        "test-return-state": {
+            "comment": "insertcommenthere",
+            "name": "test-state-1",
+            "start_time": "01: 19: 51.105566",
+            "result": True,
+            "duration": 3.645,
+            "__run_num__": 193,
+            "changes": {},
+            "__id__": "test-return-state",
+        },
+        "test-return-state2": {
+            "comment": "insertcommenthere",
+            "name": "test-state-2",
+            "start_time": "01: 19: 51.105566",
+            "result": False,
+            "duration": 3.645,
+            "__run_num__": 194,
+            "changes": {},
+            "__id__": "test-return-state",
+        },
+    },
+    "retcode": 2,
+    "success": True,
+    "fun": "state.highstate",
+    "id": "AppOptics-Test",
+    "out": "highstate",
+}
+
+
+class AppOpticsTest(ShellCase):
+    """
+    Test the AppOptics returner
+    """
+
+    def test_count_runtimes(self):
+        """
+        Test the calculations
+        """
+        results = appoptics_return._calculate_runtimes(MOCK_RET_HIGHSTATE["return"])
+        self.assertEqual(results["num_failed_states"], 1)
+        self.assertEqual(results["num_passed_states"], 1)
+        self.assertEqual(results["runtime"], 7.29)