Merge branch '2017.7' into ssh_jinja

doc/ref/modules/index.rst

@@ -376,6 +376,22 @@ The above example will force the minion to use the :py:mod:`systemd
 
 .. __: https://github.com/saltstack/salt/issues/new
 
+Logging Restrictions
+--------------------
+
+As a rule, logging should not be done anywhere in a Salt module before it is
+loaded. This rule apples to all code that would run before the ``__virtual__()``
+function, as well as the code within the ``__virtual__()`` function itself.
+
+If logging statements are made before the virtual function determines if
+the module should be loaded, then those logging statements will be called
+repeatedly. This clutters up log files unnecessarily.
+
+Exceptions may be considered for logging statements made at the ``trace`` level.
+However, it is better to provide the necessary information by another means.
+One method is to :ref:`return error information <modules-error-info>` in the
+``__virtual__()`` function.
+
 .. _modules-virtual-name:
 
 ``__virtualname__``

salt/netapi/rest_cherrypy/app.py

@@ -106,6 +106,13 @@ A REST API for Salt
     expire_responses : True
         Whether to check for and kill HTTP responses that have exceeded the
         default timeout.
+
+        .. deprecated:: 2016.11.9, 2017.7.3, Oxygen
+
+            The "expire_responses" configuration setting, which corresponds
+            to the ``timeout_monitor`` setting in CherryPy, is no longer
+            supported in CherryPy versions >= 12.0.0.
+
     max_request_body_size : ``1048576``
         Maximum size for the HTTP request body.
     collect_stats : False
@@ -506,6 +513,7 @@ import salt.ext.six as six
 # Import Salt libs
 import salt
 import salt.auth
+import salt.exceptions
 import salt.utils
 import salt.utils.event
 
@@ -753,11 +761,18 @@ def hypermedia_handler(*args, **kwargs):
     except (salt.exceptions.SaltDaemonNotRunning,
             salt.exceptions.SaltReqTimeoutError) as exc:
         raise cherrypy.HTTPError(503, exc.strerror)
-    except (cherrypy.TimeoutError, salt.exceptions.SaltClientTimeout):
+    except salt.exceptions.SaltClientTimeout:
         raise cherrypy.HTTPError(504)
     except cherrypy.CherryPyException:
         raise
     except Exception as exc:
+        # The TimeoutError exception class was removed in CherryPy in 12.0.0, but
+        # Still check existence of TimeoutError and handle in CherryPy < 12.
+        # The check was moved down from the SaltClientTimeout error line because
+        # A one-line if statement throws a BaseException inheritance TypeError.
+        if hasattr(cherrypy, 'TimeoutError') and isinstance(exc, cherrypy.TimeoutError):
+            raise cherrypy.HTTPError(504)
+
         import traceback
 
         logger.debug("Error while processing request for: %s",
@@ -2731,8 +2746,6 @@ class API(object):
                 'server.socket_port': self.apiopts.get('port', 8000),
                 'server.thread_pool': self.apiopts.get('thread_pool', 100),
                 'server.socket_queue_size': self.apiopts.get('queue_size', 30),
-                'engine.timeout_monitor.on': self.apiopts.get(
-                    'expire_responses', True),
                 'max_request_body_size': self.apiopts.get(
                     'max_request_body_size', 1048576),
                 'debug': self.apiopts.get('debug', False),
@@ -2750,6 +2763,14 @@ class API(object):
             },
         }
 
+        if salt.utils.version_cmp(cherrypy.__version__, '12.0.0') < 0:
+            # CherryPy >= 12.0 no longer supports "timeout_monitor", only set
+            # this config option when using an older version of CherryPy.
+            # See Issue #44601 for more information.
+            conf['global']['engine.timeout_monitor.on'] = self.apiopts.get(
+                'expire_responses', True
+            )
+
         if cpstats and self.apiopts.get('collect_stats', False):
             conf['/']['tools.cpstats.on'] = True
 

salt/utils/__init__.py

@@ -2016,7 +2016,11 @@ def check_state_result(running, recurse=False, highstate=None):
 
     ret = True
     for state_id, state_result in six.iteritems(running):
-        if not recurse and not isinstance(state_result, dict):
+        expected_type = dict
+        # The __extend__ state is a list
+        if "__extend__" == state_id:
+            expected_type = list
+        if not recurse and not isinstance(state_result, expected_type):
             ret = False
         if ret and isinstance(state_result, dict):
             result = state_result.get('result', _empty)

tests/integration/files/file/prod/non-base-env.sls

@@ -0,0 +1,4 @@
+test_file:
+  file.managed:
+    - name: /tmp/nonbase_env
+    - source: salt://nonbase_env

tests/integration/files/file/prod/nonbase_env

@@ -0,0 +1 @@
+it worked - new environment!

tests/integration/modules/test_state.py

@@ -1240,3 +1240,23 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin):
         self.assertIn(state_id, state_run)
         self.assertEqual(state_run[state_id]['comment'], 'Failure!')
         self.assertFalse(state_run[state_id]['result'])
+
+    def test_state_nonbase_environment(self):
+        '''
+        test state.sls with saltenv using a nonbase environment
+        with a salt source
+        '''
+        state_run = self.run_function(
+            'state.sls',
+            mods='non-base-env',
+            saltenv='prod'
+        )
+        state_id = 'file_|-test_file_|-/tmp/nonbase_env_|-managed'
+        self.assertEqual(state_run[state_id]['comment'], 'File /tmp/nonbase_env updated')
+        self.assertTrue(state_run['file_|-test_file_|-/tmp/nonbase_env_|-managed']['result'])
+        self.assertTrue(os.path.isfile('/tmp/nonbase_env'))
+
+    def tearDown(self):
+        nonbase_file = '/tmp/nonbase_env'
+        if os.path.isfile(nonbase_file):
+            os.remove(nonbase_file)

tests/integration/shell/test_key.py

@@ -5,6 +5,7 @@ from __future__ import absolute_import
 import os
 import shutil
 import tempfile
+import textwrap
 
 # Import Salt Testing libs
 from tests.support.case import ShellCase
@@ -56,6 +57,36 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin):
             if USERA in user:
                 self.run_call('user.delete {0} remove=True'.format(USERA))
 
+    def test_remove_key(self):
+        '''
+        test salt-key -d usage
+        '''
+        min_name = 'minibar'
+        pki_dir = self.master_opts['pki_dir']
+        key = os.path.join(pki_dir, 'minions', min_name)
+
+        with salt.utils.fopen(key, 'w') as fp:
+            fp.write(textwrap.dedent('''\
+                     -----BEGIN PUBLIC KEY-----
+                     MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoqIZDtcQtqUNs0wC7qQz
+                     JwFhXAVNT5C8M8zhI+pFtF/63KoN5k1WwAqP2j3LquTG68WpxcBwLtKfd7FVA/Kr
+                     OF3kXDWFnDi+HDchW2lJObgfzLckWNRFaF8SBvFM2dys3CGSgCV0S/qxnRAjrJQb
+                     B3uQwtZ64ncJAlkYpArv3GwsfRJ5UUQnYPDEJwGzMskZ0pHd60WwM1gMlfYmNX5O
+                     RBEjybyNpYDzpda6e6Ypsn6ePGLkP/tuwUf+q9wpbRE3ZwqERC2XRPux+HX2rGP+
+                     mkzpmuHkyi2wV33A9pDfMgRHdln2CLX0KgfRGixUQhW1o+Kmfv2rq4sGwpCgLbTh
+                     NwIDAQAB
+                     -----END PUBLIC KEY-----
+                     '''))
+
+        check_key = self.run_key('-p {0}'.format(min_name))
+        self.assertIn('Accepted Keys:', check_key)
+        self.assertIn('minibar:  -----BEGIN PUBLIC KEY-----', check_key)
+
+        remove_key = self.run_key('-d {0} -y'.format(min_name))
+
+        check_key = self.run_key('-p {0}'.format(min_name))
+        self.assertEqual([], check_key)
+
     def test_list_accepted_args(self):
         '''
         test salt-key -l for accepted arguments

tests/integration/ssh/test_deploy.py

@@ -4,6 +4,8 @@ salt-ssh testing
 '''
 # Import Python libs
 from __future__ import absolute_import
+import os
+import shutil
 
 # Import salt testing libs
 from tests.support.case import SSHCase
@@ -19,3 +21,21 @@ class SSHTest(SSHCase):
         '''
         ret = self.run_function('test.ping')
         self.assertTrue(ret, 'Ping did not return true')
+
+    def test_thin_dir(self):
+        '''
+        test to make sure thin_dir is created
+        and salt-call file is included
+        '''
+        thin_dir = self.run_function('config.get', ['thin_dir'], wipe=False)
+        os.path.isdir(thin_dir)
+        os.path.exists(os.path.join(thin_dir, 'salt-call'))
+        os.path.exists(os.path.join(thin_dir, 'running_data'))
+
+    def tearDown(self):
+        '''
+        make sure to clean up any old ssh directories
+        '''
+        salt_dir = self.run_function('config.get', ['thin_dir'], wipe=False)
+        if os.path.exists(salt_dir):
+            shutil.rmtree(salt_dir)