Merge branch 'master' into win_task_repeat

.pylintrc

@@ -675,7 +675,8 @@ allowed-3rd-party-modules=msgpack,
                           libcloud,
                           zmq,
                           pytest,
-                          pytestsalt
+                          pytestsalt,
+                          setuptools
 
 
 [EXCEPTIONS]

doc/topics/releases/3000.rst

@@ -19,6 +19,10 @@ compare Salt's versions, please use LooseVersion. There is also the packaging
 library you can use to compare versions. Another alternative is using the
 :py:func:`salt version module<salt.modules.salt_version>`
 
+Python 3.8 Not Supported
+==========================
+Please note that running Salt with Python 3.8 is currently not supported.
+It is recommended to not use a version higher than 3.7.
 
 Vendored Tornado Code Base
 ==========================
@@ -29,6 +33,12 @@ we continue to work on upgrading Tornado to a recent version. New code that
 uses Tornado should import the module from ``salt.ext.tornado`` instead of
 importing the system version of Tornado.
 
+Msgpack 1.0.0rc1 Incompatibility
+================================
+Salt is currently incompatible with mgspack 1.0.0rc1. We recommend using
+versions < 1.0.0 . This impacts the Salt Master's ability to run commands
+against minions. Please see `Issue 56007 <https://github.com/saltstack/salt/issues/56007>`_
+for more details and updates.
 
 Saltcheck Updates
 =================

pkg/windows/req.txt

@@ -38,8 +38,3 @@ smmap==0.9.0
 timelib==0.2.4
 watchdog==0.9.0
 wheel==0.33.4
-# Requirements for Tornado 4.5.3 (vendored as salt.ext.tornado)
-backports-abc==0.5
-singledispatch==3.4.0.3
-# Required by Tornado to handle threads stuff.
-futures>=2.0; python_version < '3.0'

requirements/base.txt

@@ -1,7 +1,5 @@
 Jinja2
-# This should be changed to msgpack-python for Packages
-# msgpack-python>0.3,!=0.5.5
-msgpack>=0.5,!=0.5.5
+msgpack>=0.5,!=0.5.5,<1.0.0
 PyYAML
 MarkupSafe
 requests>=1.0.0

salt/cache/consul.py

@@ -15,7 +15,7 @@ The related documentation can be found in the `Consul documentation`_.
 To enable this cache plugin, the master will need the python client for
 Consul installed. This can be easily installed with pip:
 
-.. code-block: bash
+.. code-block:: bash
 
     pip install python-consul
 

salt/cache/etcd_cache.py

@@ -13,7 +13,7 @@ The related documentation can be found in the `Etcd documentation`_.
 To enable this cache plugin, the master will need the python client for
 Etcd installed. This can be easily installed with pip:
 
-.. code-block: bash
+.. code-block:: bash
 
     pip install python-etcd
 

salt/cache/mysql_cache.py

@@ -13,7 +13,7 @@ table if needed. The keys are indexed using the `bank` and `etcd_key` columns.
 To enable this cache plugin, the master will need the python client for
 MySQL installed. This can be easily installed with pip:
 
-.. code-block: bash
+.. code-block:: bash
 
     pip install python-mysql
 

salt/modules/bamboohr.py

@@ -6,7 +6,7 @@ Support for BambooHR
 
 Requires a ``subdomain`` and an ``apikey`` in ``/etc/salt/minion``:
 
-.. code-block: yaml
+.. code-block:: yaml
 
     bamboohr:
       apikey: 012345678901234567890

salt/modules/ifttt.py

@@ -6,7 +6,7 @@ Support for IFTTT
 
 Requires an ``api_key`` in ``/etc/salt/minion``:
 
-.. code-block: yaml
+.. code-block:: yaml
 
     ifttt:
       secret_key: '280d4699-a817-4719-ba6f-ca56e573e44f'

salt/modules/pushbullet.py

@@ -6,7 +6,7 @@ Module for sending messages to Pushbullet (https://www.pushbullet.com)
 
 Requires an ``api_key`` in ``/etc/salt/minion``:
 
-.. code-block: yaml
+.. code-block:: yaml
 
     pushbullet:
       api_key: 'ABC123abc123ABC123abc123ABC123ab'

salt/modules/rallydev.py

@@ -6,7 +6,7 @@ Support for RallyDev
 
 Requires a ``username`` and a ``password`` in ``/etc/salt/minion``:
 
-.. code-block: yaml
+.. code-block:: yaml
 
     rallydev:
       username: myuser@example.com

salt/modules/vault.py

@@ -74,7 +74,7 @@ Functions to interact with Hashicorp Vault.
         You can still use the token auth via a OS environment variable via this
         config example:
 
-        .. code-block: yaml
+        .. code-block:: yaml
 
            vault:
              url: https://vault.service.domain:8200
@@ -86,7 +86,8 @@ Functions to interact with Hashicorp Vault.
 
         And then export the VAULT_TOKEN variable in your OS:
 
-        .. code-block: bash
+        .. code-block:: bash
+
            export VAULT_TOKEN=11111111-1111-1111-1111-1111111111111
 
     policies
@@ -100,7 +101,7 @@ Functions to interact with Hashicorp Vault.
         expanded into multiple policies. For example, given the template
         ``saltstack/by-role/{grains[roles]}``, and a minion having these grains:
 
-        .. code-block: yaml
+        .. code-block:: yaml
 
             grains:
                 roles:

salt/modules/vboxmanage.py

@@ -7,7 +7,7 @@ Support for VirtualBox using the VBoxManage command
 If the ``vboxdrv`` kernel module is not loaded, this module can automatically
 load it by configuring ``autoload_vboxdrv`` in ``/etc/salt/minion``:
 
-.. code-block: yaml
+.. code-block:: yaml
 
     autoload_vboxdrv: True
 

salt/modules/victorops.py

@@ -6,7 +6,7 @@ Support for VictorOps
 
 Requires an ``api_key`` in ``/etc/salt/minion``:
 
-.. code-block: yaml
+.. code-block:: yaml
 
     victorops:
       api_key: '280d4699-a817-4719-ba6f-ca56e573e44f'

salt/modules/vsphere.py

@@ -7906,7 +7906,7 @@ def _apply_scsi_controller(adapter, adapter_type, bus_sharing, key,
         Describes the operation which should be done on the object,
         the possibles values: 'add' and 'edit', the default value is 'add'
 
-    .. code-block: bash
+    .. code-block:: bash
 
         scsi:
           adapter: 'SCSI controller 0'
@@ -8097,7 +8097,7 @@ def _apply_cd_drive(drive_label, key, device_type, operation,
     parent_ref
         Parent object
 
-    .. code-block: bash
+    .. code-block:: bash
 
         cd:
             adapter: "CD/DVD drive 1"
@@ -8278,7 +8278,7 @@ def _create_disks(service_instance, disks, scsi_controllers=None, parent=None):
     parent
         Parent object reference
 
-    .. code-block: bash
+    .. code-block:: bash
 
         disk:
           adapter: 'Hard disk 1'
@@ -8389,7 +8389,7 @@ def _create_network_adapters(network_interfaces, parent=None):
     parent
         Parent object reference
 
-    .. code-block: bash
+    .. code-block:: bash
 
         interfaces:
           adapter: 'Network adapter 1'

salt/modules/zk_concurrency.py

@@ -185,7 +185,7 @@ def lock_holders(path,
 
     Example:
 
-    .. code-block: bash
+    .. code-block:: bash
 
         salt minion zk_concurrency.lock_holders /lock/path host1:1234,host2:1234
     '''
@@ -237,7 +237,7 @@ def lock(path,
 
     Example:
 
-    .. code-block: bash
+    .. code-block:: bash
 
         salt minion zk_concurrency.lock /lock/path host1:1234,host2:1234
     '''
@@ -298,7 +298,7 @@ def unlock(path,
 
     Example:
 
-    .. code-block: bash
+    .. code-block:: bash
 
         salt minion zk_concurrency.unlock /lock/path host1:1234,host2:1234
     '''
@@ -348,7 +348,7 @@ def party_members(path,
 
     Example:
 
-    .. code-block: bash
+    .. code-block:: bash
 
         salt minion zk_concurrency.party_members /lock/path host1:1234,host2:1234
         salt minion zk_concurrency.party_members /lock/path host1:1234,host2:1234 min_nodes=3 blocking=True

salt/proxy/nxos.py

@@ -163,7 +163,7 @@ def ping():
     '''
     Ping the device on the other end of the connection
 
-    .. code-block: bash
+    .. code-block:: bash
 
         salt '*' nxos.cmd ping
     '''
@@ -187,7 +187,7 @@ def sendline(command):
     '''
     Run command through switch's cli
 
-    .. code-block: bash
+    .. code-block:: bash
 
         salt '*' nxos.cmd sendline 'show run | include "^username admin password"'
     '''
@@ -203,7 +203,7 @@ def grains():
     '''
     Get grains for proxy minion
 
-    .. code-block: bash
+    .. code-block:: bash
 
         salt '*' nxos.cmd grains
     '''
@@ -218,7 +218,7 @@ def grains_refresh():
     '''
     Refresh the grains from the proxy device.
 
-    .. code-block: bash
+    .. code-block:: bash
 
         salt '*' nxos.cmd grains_refresh
     '''
@@ -230,7 +230,7 @@ def get_user(username):
     '''
     Get username line from switch
 
-    .. code-block: bash
+    .. code-block:: bash
 
         salt '*' nxos.cmd get_user username=admin
     '''
@@ -241,7 +241,7 @@ def get_roles(username):
     '''
     Get roles that the username is assigned from switch
 
-    .. code-block: bash
+    .. code-block:: bash
 
         salt '*' nxos.cmd get_roles username=admin
     '''
@@ -258,7 +258,7 @@ def check_password(username, password, encrypted=False):
     '''
     Check if passed password is the one assigned to user
 
-    .. code-block: bash
+    .. code-block:: bash
 
         salt '*' nxos.cmd check_password username=admin password=admin
         salt '*' nxos.cmd check_password username=admin \\

salt/runners/drac.py

@@ -5,7 +5,7 @@ Manage Dell DRAC from the Master
 The login credentials need to be configured in the Salt master
 configuration file.
 
-.. code-block: yaml
+.. code-block:: yaml
 
     drac:
       username: admin

salt/sdb/sqlite3.py

@@ -31,7 +31,7 @@ Advanced Usage:
 Instead of a table name, it is possible to provide custom SQL statements to
 create the table(s) and get and set values.
 
-.. code-block: yaml
+.. code-block:: yaml
 
     myadvanced
       driver: sqlite3

salt/states/chocolatey.py

@@ -215,7 +215,7 @@ def uninstalled(name, version=None, uninstall_args=None, override_args=False):
       When this is set to False uninstall_args will be appended to the end of
       the default arguments
 
-    .. code-block: yaml
+    .. code-block:: yaml
 
       Removemypackage:
         chocolatey.uninstalled:

salt/states/lxd_image.py

@@ -70,7 +70,7 @@ def present(name,
 
         For an LXD to LXD copy:
 
-        .. code-block: yaml
+        .. code-block:: yaml
 
             source:
                 type: lxd
@@ -89,7 +89,7 @@ def present(name,
 
         From file:
 
-        .. code-block: yaml
+        .. code-block:: yaml
 
             source:
                 type: file
@@ -98,7 +98,7 @@ def present(name,
 
         From simplestreams:
 
-        .. code-block: yaml
+        .. code-block:: yaml
 
             source:
                 type: simplestreams
@@ -107,7 +107,7 @@ def present(name,
 
         From an URL:
 
-        .. code-block: yaml
+        .. code-block:: yaml
 
             source:
                 type: url

salt/thorium/file.py

@@ -5,14 +5,14 @@ Writes matches to disk to verify activity, helpful when testing
 Normally this is used by giving the name of the file (without a path) that the
 data will be saved to. If for instance you use ``foo`` as the name:
 
-.. code-block: yaml
+.. code-block:: yaml
 
     foo:
       file.save
 
 Then the file will be saved to:
 
-.. code-block: bash
+.. code-block:: bash
 
     <salt cachedir>/thorium/saves/foo
 

salt/utils/aws.py

@@ -383,7 +383,7 @@ def query(params=None, setname=None, requesturl=None, location=None,
 
     Default ``product`` is ``ec2``. Valid ``product`` names are:
 
-    .. code-block: yaml
+    .. code-block:: yaml
 
         - autoscaling (Auto Scaling)
         - cloudformation (CloudFormation)

salt/utils/cloud.py

@@ -3137,7 +3137,7 @@ def _strip_cache_events(data, opts):
     are configured in the main Salt Cloud configuration file, usually
     ``/etc/salt/cloud``.
 
-    .. code-block: yaml
+    .. code-block:: yaml
 
         cache_event_strip_fields:
           - password

setup.py

@@ -5,7 +5,7 @@ The setup script for salt
 '''
 
 # pylint: disable=file-perms,ungrouped-imports,wrong-import-order,wrong-import-position,repr-flag-used-in-string
-# pylint: disable=3rd-party-local-module-not-gated,resource-leakage
+# pylint: disable=3rd-party-local-module-not-gated,resource-leakage,blacklisted-module
 # pylint: disable=C0111,E1101,E1103,F0401,W0611,W0201,W0232,R0201,R0902,R0903
 
 # For Python 2.5.  A no-op on 2.6 and above.
@@ -14,6 +14,7 @@ from __future__ import absolute_import, print_function, with_statement
 import os
 import sys
 import glob
+import inspect
 import operator
 import platform
 try:
@@ -22,16 +23,21 @@ except ImportError:
     from urllib.request import urlopen  # pylint: disable=no-name-in-module
 from datetime import datetime
 # pylint: disable=E0611
+import setuptools
 import distutils.dist
 from distutils import log
 from distutils.cmd import Command
 from distutils.errors import DistutilsArgError
 from distutils.command.build import build
 from distutils.command.clean import clean
-from distutils.command.sdist import sdist
 from distutils.command.install_lib import install_lib
 from distutils.version import LooseVersion  # pylint: disable=blacklisted-module
 from ctypes.util import find_library
+from setuptools import setup
+from setuptools.command.develop import develop
+from setuptools.command.install import install
+from setuptools.command.sdist import sdist
+from setuptools.command.egg_info import egg_info
 # pylint: enable=E0611
 
 try:
@@ -78,36 +84,6 @@ else:
 # Store a reference whether if we're running under Python 3 and above
 IS_PY3 = sys.version_info > (3,)
 
-# Use setuptools only if the user opts-in by setting the USE_SETUPTOOLS env var
-# Or if setuptools was previously imported (which is the case when using
-# 'distribute')
-# This ensures consistent behavior but allows for advanced usage with
-# virtualenv, buildout, and others.
-WITH_SETUPTOOLS = False
-if 'USE_SETUPTOOLS' in os.environ or 'setuptools' in sys.modules:
-    try:
-        from setuptools import setup
-        from setuptools.command.develop import develop
-        from setuptools.command.install import install
-        from setuptools.command.sdist import sdist
-        from setuptools.command.egg_info import egg_info
-        WITH_SETUPTOOLS = True
-    except ImportError:
-        WITH_SETUPTOOLS = False
-
-if WITH_SETUPTOOLS is False:
-    import warnings
-    # pylint: disable=E0611
-    from distutils.command.install import install
-    from distutils.core import setup
-    # pylint: enable=E0611
-    warnings.filterwarnings(
-        'ignore',
-        'Unknown distribution option: \'(extras_require|tests_require|install_requires|zip_safe)\'',
-        UserWarning,
-        'distutils.dist'
-    )
-
 try:
     # Add the esky bdist target if the module is available
     # may require additional modules depending on platform
@@ -345,70 +321,69 @@ class WriteSaltSshPackagingFile(Command):
             # pylint: enable=E0602
 
 
-if WITH_SETUPTOOLS:
-    class Develop(develop):
-        user_options = develop.user_options + [
-            ('write-salt-version', None,
-             'Generate Salt\'s _version.py file which allows proper version '
-             'reporting. This defaults to False on develop/editable setups. '
-             'If WRITE_SALT_VERSION is found in the environment this flag is '
-             'switched to True.'),
-            ('generate-salt-syspaths', None,
-             'Generate Salt\'s _syspaths.py file which allows tweaking some '
-             'common paths that salt uses. This defaults to False on '
-             'develop/editable setups. If GENERATE_SALT_SYSPATHS is found in '
-             'the environment this flag is switched to True.'),
-            ('mimic-salt-install', None,
-             'Mimmic the install command when running the develop command. '
-             'This will generate salt\'s _version.py and _syspaths.py files. '
-             'Generate Salt\'s _syspaths.py file which allows tweaking some '
-             'This defaults to False on develop/editable setups. '
-             'If MIMIC_INSTALL is found in the environment this flag is '
-             'switched to True.')
-        ]
-        boolean_options = develop.boolean_options + [
-            'write-salt-version',
-            'generate-salt-syspaths',
-            'mimic-salt-install'
-        ]
+class Develop(develop):
+    user_options = develop.user_options + [
+        ('write-salt-version', None,
+         'Generate Salt\'s _version.py file which allows proper version '
+         'reporting. This defaults to False on develop/editable setups. '
+         'If WRITE_SALT_VERSION is found in the environment this flag is '
+         'switched to True.'),
+        ('generate-salt-syspaths', None,
+         'Generate Salt\'s _syspaths.py file which allows tweaking some '
+         'common paths that salt uses. This defaults to False on '
+         'develop/editable setups. If GENERATE_SALT_SYSPATHS is found in '
+         'the environment this flag is switched to True.'),
+        ('mimic-salt-install', None,
+         'Mimmic the install command when running the develop command. '
+         'This will generate salt\'s _version.py and _syspaths.py files. '
+         'Generate Salt\'s _syspaths.py file which allows tweaking some '
+         'This defaults to False on develop/editable setups. '
+         'If MIMIC_INSTALL is found in the environment this flag is '
+         'switched to True.')
+    ]
+    boolean_options = develop.boolean_options + [
+        'write-salt-version',
+        'generate-salt-syspaths',
+        'mimic-salt-install'
+    ]
 
-        def initialize_options(self):
-            develop.initialize_options(self)
-            self.write_salt_version = False
-            self.generate_salt_syspaths = False
-            self.mimic_salt_install = False
+    def initialize_options(self):
+        develop.initialize_options(self)
+        self.write_salt_version = False
+        self.generate_salt_syspaths = False
+        self.mimic_salt_install = False
 
-        def finalize_options(self):
-            develop.finalize_options(self)
-            if 'WRITE_SALT_VERSION' in os.environ:
-                self.write_salt_version = True
-            if 'GENERATE_SALT_SYSPATHS' in os.environ:
-                self.generate_salt_syspaths = True
-            if 'MIMIC_SALT_INSTALL' in os.environ:
-                self.mimic_salt_install = True
+    def finalize_options(self):
+        develop.finalize_options(self)
+        if 'WRITE_SALT_VERSION' in os.environ:
+            self.write_salt_version = True
+        if 'GENERATE_SALT_SYSPATHS' in os.environ:
+            self.generate_salt_syspaths = True
+        if 'MIMIC_SALT_INSTALL' in os.environ:
+            self.mimic_salt_install = True
 
-            if self.mimic_salt_install:
-                self.write_salt_version = True
-                self.generate_salt_syspaths = True
+        if self.mimic_salt_install:
+            self.write_salt_version = True
+            self.generate_salt_syspaths = True
 
-        def run(self):
-            if IS_WINDOWS_PLATFORM:
-                # Download the required DLLs
-                self.distribution.salt_download_windows_dlls = True
-                self.run_command('download-windows-dlls')
-                self.distribution.salt_download_windows_dlls = None
+    def run(self):
+        if IS_WINDOWS_PLATFORM:
+            # Download the required DLLs
+            self.distribution.salt_download_windows_dlls = True
+            self.run_command('download-windows-dlls')
+            self.distribution.salt_download_windows_dlls = None
 
-            if self.write_salt_version is True:
-                self.distribution.running_salt_install = True
-                self.distribution.salt_version_hardcoded_path = SALT_VERSION_HARDCODED
-                self.run_command('write_salt_version')
+        if self.write_salt_version is True:
+            self.distribution.running_salt_install = True
+            self.distribution.salt_version_hardcoded_path = SALT_VERSION_HARDCODED
+            self.run_command('write_salt_version')
 
-            if self.generate_salt_syspaths:
-                self.distribution.salt_syspaths_hardcoded_path = SALT_SYSPATHS_HARDCODED
-                self.run_command('generate_salt_syspaths')
+        if self.generate_salt_syspaths:
+            self.distribution.salt_syspaths_hardcoded_path = SALT_SYSPATHS_HARDCODED
+            self.run_command('generate_salt_syspaths')
 
-            # Resume normal execution
-            develop.run(self)
+        # Resume normal execution
+        develop.run(self)
 
 
 class DownloadWindowsDlls(Command):
@@ -724,6 +699,15 @@ class Install(install):
         install.finalize_options(self)
 
     def run(self):
+        from distutils.version import StrictVersion
+        if StrictVersion(setuptools.__version__) < StrictVersion('9.1'):
+            sys.stderr.write(
+                '\n\nInstalling Salt requires setuptools >= 9.1\n'
+                'Available setuptools version is {}\n\n'.format(setuptools.__version__)
+            )
+            sys.stderr.flush()
+            sys.exit(1)
+
         # Let's set the running_salt_install attribute so we can add
         # _version.py in the build command
         self.distribution.running_salt_install = True
@@ -738,6 +722,38 @@ class Install(install):
         # Run install.run
         install.run(self)
 
+    @staticmethod
+    def _called_from_setup(run_frame):
+        """
+        Attempt to detect whether run() was called from setup() or by another
+        command.  If called by setup(), the parent caller will be the
+        'run_command' method in 'distutils.dist', and *its* caller will be
+        the 'run_commands' method.  If called any other way, the
+        immediate caller *might* be 'run_command', but it won't have been
+        called by 'run_commands'. Return True in that case or if a call stack
+        is unavailable. Return False otherwise.
+        """
+        if run_frame is None:
+            # If run_frame is None, just call the parent class logic
+            return install._called_from_setup(run_frame)
+
+        # Because Salt subclasses the setuptools install command, it needs to
+        # override this static method to provide the right frame for the logic
+        # so apply.
+
+        # We first try the current run_frame in case the issue
+        # https://github.com/pypa/setuptools/issues/456 is fixed.
+        first_call = install._called_from_setup(run_frame)
+        if first_call:
+            return True
+
+        # Fallback to providing the parent frame to have the right logic kick in
+        second_call = install._called_from_setup(run_frame.f_back)
+        if second_call is None:
+            # There was no parent frame?!
+            return first_call
+        return second_call
+
 
 class InstallLib(install_lib):
     def run(self):
@@ -875,6 +891,7 @@ class SaltDistribution(distutils.dist.Distribution):
                               'build': Build,
                               'sdist': Sdist,
                               'install': Install,
+                              'develop': Develop,
                               'write_salt_version': WriteSaltVersion,
                               'generate_salt_syspaths': GenerateSaltSyspaths,
                               'write_salt_ssh_packaging_file': WriteSaltSshPackagingFile})
@@ -884,9 +901,6 @@ class SaltDistribution(distutils.dist.Distribution):
         if IS_WINDOWS_PLATFORM:
             self.cmdclass.update({'download-windows-dlls': DownloadWindowsDlls})
 
-        if WITH_SETUPTOOLS:
-            self.cmdclass.update({'develop': Develop})
-
         self.license = 'Apache Software License 2.0'
         self.packages = self.discover_packages()
         self.zip_safe = False