Merge pull request #51148 from Ch3LL/merge-2019.2

[2019.2] Merge forward from 2018.3 to 2019.2

doc/ref/cli/_includes/output-options.rst

@@ -6,12 +6,9 @@ Output Options
     Pass in an alternative outputter to display the return of data. This
     outputter can be any of the available outputters:
 
-        ``grains``, ``highstate``, ``json``, ``key``, ``overstatestage``, ``pprint``, ``raw``, ``txt``, ``yaml``
-
-    Some outputters are formatted only for data returned from specific
-    functions; for instance, the ``grains`` outputter will not work for non-grains
-    data.
+        ``highstate``, ``json``, ``key``, ``overstatestage``, ``pprint``, ``raw``, ``txt``, ``yaml``, and :ref:`many others <all-salt.output>`.
 
+    Some outputters are formatted only for data returned from specific functions.
     If an outputter is used that does not support the data passed into it, then
     Salt will fall back on the ``pprint`` outputter and display the return data
     using the Python ``pprint`` standard library module.

doc/ref/output/all/index.rst

@@ -22,6 +22,7 @@ Follow one of the below links for further information and examples
     overstatestage
     pony
     pprint_out
+    profile
     progress
     raw
     table_out

doc/ref/output/all/salt.output.profile.rst

@@ -0,0 +1,6 @@
+===================
+salt.output.profile
+===================
+
+.. automodule:: salt.output.profile
+    :members:

doc/topics/troubleshooting/master.rst

@@ -219,6 +219,10 @@ configuration file: ``/etc/salt/master`` and setting the ``timeout`` value to
 change the default timeout for all commands, and then restarting the
 salt-master service.
 
+If a ``state.apply`` run takes too long, you can find a bottleneck by adding the
+:py:mod:`--out=profile <salt.output.profile>` option.
+
+
 Salt Master Auth Flooding
 =========================
 

doc/topics/troubleshooting/minion.rst

@@ -152,3 +152,6 @@ salt-minion service.
     Modifying the minion timeout value is not required when running commands
     from a Salt Master. It is only required when running commands locally on
     the minion.
+
+If a ``state.apply`` run takes too long, you can find a bottleneck by adding the
+:py:mod:`--out=profile <salt.output.profile>` option.

salt/modules/mysql.py

@@ -808,7 +808,7 @@ def version(**connection_args):
         return ''
 
     try:
-        return cur.fetchone()[0]
+        return salt.utils.data.decode(cur.fetchone()[0])
     except IndexError:
         return ''
 

salt/modules/nxos.py

@@ -59,7 +59,7 @@ def cmd(command, *args, **kwargs):
     proxy_cmd = '.'.join([proxy_prefix, command])
     if proxy_cmd not in __proxy__:
         return False
-    for k in kwargs:
+    for k in list(kwargs):
         if k.startswith('__pub_'):
             kwargs.pop(k)
     return __proxy__[proxy_cmd](*args, **kwargs)

salt/output/profile.py

@@ -1,4 +1,31 @@
 # -*- coding: utf-8 -*-
+'''
+Display profiling data in a table format
+========================================
+
+Show profile data for returners that would normally show a highstate output.
+
+    salt MINION state.apply something --out=profile
+
+Attempt to output the returns of state.sls and state.highstate as a table of
+names, modules and durations that looks somewhat like the following::
+
+    name                mod.fun                duration (ms)
+    --------------------------------------------------------
+    I-fail-unless-stmt  other.function               -1.0000
+    old-minion-config   grains.list_present           1.1200
+    salt-data           group.present                48.3800
+    /etc/salt/minion    file.managed                 63.1450
+
+
+To get the above appearance, use settings something like these::
+
+    out.table.separate_rows: False
+    out.table.justify: left
+    out.table.delim: '  '
+    out.table.prefix: ''
+    out.table.suffix: ''
+'''
 from __future__ import absolute_import, print_function, unicode_literals
 import salt.output.table_out as table_out
 
@@ -39,28 +66,7 @@ def _find_durations(data, name_max=60):
 
 def output(data, **kwargs):
     '''
-
-    Show profile data for returners that would normally show a highstate output.
-
-        salt globhere state.sls something --out=profile
-
-    Attempt to output the returns of state.sls and state.highstate as a table of
-    names, modules and durations that looks somewhat like the following:
-
-        name                mod.fun                duration (ms)
-        --------------------------------------------------------
-        I-fail-unless-stmt  other.function               -1.0000
-        old-minion-config   grains.list_present           1.1200
-        salt-data           group.present                48.3800
-        /etc/salt/minion    file.managed                 63.1450
-
-
-    To get the above appearance, use settings something like these:
-      out.table.separate_rows: False
-      out.table.justify: left
-      out.table.delim: '  '
-      out.table.prefix: ''
-      out.table.suffix: ''
+    Display the profiling data in a table format.
     '''
 
     rows = _find_durations(data)

salt/states/elasticsearch.py

@@ -234,7 +234,7 @@ def index_template_absent(name):
 
 def index_template_present(name, definition, check_definition=False):
     '''
-    Ensure that the named index templat eis present.
+    Ensure that the named index template is present.
 
     name
         Name of the index to add
@@ -248,7 +248,7 @@ def index_template_present(name, definition, check_definition=False):
     .. code-block:: yaml
 
         mytestindex2_template:
-          elasticsearch_index_template.present:
+          elasticsearch.index_template_present:
             - definition:
                 template: logstash-*
                 order: 1

tests/unit/states/test_git.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+'''
+    :codeauthor: Erik Johnson <erik@saltstack.com>
+'''
+
+# Import Python libs
+from __future__ import absolute_import
+import logging
+import os
+
+# Import Salt Testing Libs
+from tests.support.helpers import with_tempdir
+from tests.support.mixins import LoaderModuleMockMixin
+from tests.support.unit import TestCase, skipIf
+from tests.support.mock import (
+    Mock,
+    MagicMock,
+    patch,
+    DEFAULT,
+    NO_MOCK,
+    NO_MOCK_REASON,
+)
+
+# Import Salt Libs
+import salt.states.git as git_state  # Don't potentially shadow GitPython
+
+log = logging.getLogger(__name__)
+
+
+@skipIf(NO_MOCK, NO_MOCK_REASON)
+class GitTestCase(TestCase, LoaderModuleMockMixin):
+    '''
+    Test cases for salt.states.git
+    '''
+    def setup_loader_modules(self):
+        return {
+            git_state: {
+                '__env__': 'base',
+                '__opts__': {'test': False},
+                '__salt__': {},
+            }
+        }
+
+    @with_tempdir()
+    def test_latest_no_diff_for_bare_repo(self, target):
+        '''
+        This test ensures that we don't attempt to diff when cloning a repo
+        using either bare=True or mirror=True.
+        '''
+        name = 'https://foo.com/bar/baz.git'
+        gitdir = os.path.join(target, 'refs')
+        isdir_mock = MagicMock(
+            side_effect=lambda path: DEFAULT if path != gitdir else True)
+
+        branches = ['foo', 'bar', 'baz']
+        tags = ['v1.1.0', 'v.1.1.1', 'v1.2.0']
+        local_head = 'b9ef06ab6b7524eb7c27d740dbbd5109c6d75ee4'
+        remote_head = 'eef672c1ec9b8e613905dbcd22a4612e31162807'
+
+        git_diff = Mock()
+        dunder_salt = {
+            'git.current_branch': MagicMock(return_value=branches[0]),
+            'git.diff': git_diff,
+            'git.fetch': MagicMock(return_value={}),
+            'git.is_worktree': MagicMock(return_value=False),
+            'git.list_branches': MagicMock(return_value=branches),
+            'git.list_tags': MagicMock(return_value=tags),
+            'git.remote_refs': MagicMock(return_value={'HEAD': remote_head}),
+            'git.remotes': MagicMock(return_value={
+                'origin': {'fetch': name, 'push': name},
+            }),
+            'git.rev_parse': MagicMock(side_effect=git_state.CommandExecutionError()),
+            'git.revision': MagicMock(return_value=local_head),
+            'git.version': MagicMock(return_value='1.8.3.1'),
+        }
+        with patch('os.path.isdir', isdir_mock), \
+                patch.dict(git_state.__salt__, dunder_salt):
+            result = git_state.latest(
+                name=name,
+                target=target,
+                mirror=True,  # mirror=True implies bare=True
+            )
+            assert result['result'] is True, result
+            git_diff.assert_not_called()

tests/unit/states/test_user.py

@@ -240,6 +240,7 @@ class UserTestCase(TestCase, LoaderModuleMockMixin):
                        'shadow.default_hash': shadow_hash,
                        'file.group_to_gid': MagicMock(side_effect=['foo']),
                        'file.gid_to_group': MagicMock(side_effect=[5000])}
+
         def mock_exists(*args):
             return True
 

tests/unit/test_crypt.py

@@ -8,7 +8,7 @@ import shutil
 
 # salt testing libs
 from tests.support.unit import TestCase, skipIf
-from tests.support.mock import(
+from tests.support.mock import (
     patch,
     mock_open,
     NO_MOCK,