Merge remote branch 'upstream/2016.11' into 2016.11_43417_Backport_and_Fixes

doc/conf.py

@@ -240,8 +240,8 @@ on_saltstack = 'SALT_ON_SALTSTACK' in os.environ
 project = 'Salt'
 
 version = salt.version.__version__
-latest_release = '2017.7.1'  # latest release
-previous_release = '2016.11.7'  # latest release from previous branch
+latest_release = '2017.7.2'  # latest release
+previous_release = '2016.11.8'  # latest release from previous branch
 previous_release_dir = '2016.11'  # path on web server for previous branch
 next_release = ''  # next release
 next_release_dir = ''  # path on web server for next release branch

doc/ref/configuration/minion.rst

@@ -1164,7 +1164,7 @@ be able to execute a certain module. The ``sys`` module is built into the minion
 and cannot be disabled.
 
 This setting can also tune the minion. Because all modules are loaded into system
-memory, disabling modules will lover the minion's memory footprint.
+memory, disabling modules will lower the minion's memory footprint.
 
 Modules should be specified according to their file name on the system and not by
 their virtual name. For example, to disable ``cmd``, use the string ``cmdmod`` which

doc/topics/cloud/scaleway.rst

@@ -40,7 +40,7 @@ Set up an initial profile at /etc/salt/cloud.profiles or in the /etc/salt/cloud.
 
 .. code-block:: yaml
 
-    scalewa-ubuntu:
+    scaleway-ubuntu:
       provider: my-scaleway-config
       image: Ubuntu Trusty (14.04 LTS)
 

doc/topics/eauth/index.rst

@@ -218,6 +218,7 @@ Server configuration values and their defaults:
 
     # Bind to LDAP anonymously to determine group membership
     # Active Directory does not allow anonymous binds without special configuration
+    # In addition, if auth.ldap.anonymous is True, empty bind passwords are not permitted.
     auth.ldap.anonymous: False
 
     # FOR TESTING ONLY, this is a VERY insecure setting.
@@ -250,7 +251,11 @@ and groups, it re-authenticates as the user running the Salt commands.
 
 If you are already aware of the structure of your DNs and permissions in your LDAP store are set such that
 users can look up their own group memberships, then the first and second users can be the same.  To tell Salt this is
-the case, omit the ``auth.ldap.bindpw`` parameter.  You can template the ``binddn`` like this:
+the case, omit the ``auth.ldap.bindpw`` parameter.  Note this is not the same thing as using an anonymous bind.
+Most LDAP servers will not permit anonymous bind, and as mentioned above, if `auth.ldap.anonymous` is False you
+cannot use an empty password.
+
+You can template the ``binddn`` like this:
 
 .. code-block:: yaml
 

doc/topics/releases/2016.11.8.rst

@@ -4,9 +4,21 @@ Salt 2016.11.8 Release Notes
 
 Version 2016.11.8 is a bugfix release for :ref:`2016.11.0 <release-2016-11-0>`.]
 
+Anonymous Binds and LDAP/Active Directory
+-----------------------------------------
+
+When auth.ldap.anonymous is set to False, the bind password can no longer be empty.
+
 Changes for v2016.11.7..v2016.11.8
 ----------------------------------
 
+Security Fix
+============
+
+CVE-2017-14695 Directory traversal vulnerability in minion id validation in SaltStack. Allows remote minions with incorrect credentials to authenticate to a master via a crafted minion ID. Credit for discovering the security flaw goes to: Julian Brost (julian@0x4a42.net)
+
+CVE-2017-14696 Remote Denial of Service with a specially crafted authentication request. Credit for discovering the security flaw goes to: Julian Brost (julian@0x4a42.net)
+
 Extended changelog courtesy of Todd Stansell (https://github.com/tjstansell/salt-changelogs):
 
 *Generated at: 2017-09-11T14:52:27Z*

doc/topics/releases/2016.11.9.rst

@@ -0,0 +1,6 @@
+============================
+Salt 2016.11.9 Release Notes
+============================
+
+Version 2016.11.9 is a bugfix release for :ref:`2016.11.0 <release-2016-11-0>`.]
+

doc/topics/releases/2016.3.8.rst

@@ -7,23 +7,9 @@ Version 2016.3.8 is a bugfix release for :ref:`2016.3.0 <release-2016-3-0>`.
 Changes for v2016.3.7..v2016.3.8
 --------------------------------
 
-New master configuration option `allow_minion_key_revoke`, defaults to True.  This option
-controls whether a minion can request that the master revoke its key.  When True, a minion
-can request a key revocation and the master will comply.  If it is False, the key will not
-be revoked by the msater.
+Security Fix
+============
 
-New master configuration option `require_minion_sign_messages`
-This requires that minions cryptographically sign the messages they
-publish to the master.  If minions are not signing, then log this information
-at loglevel 'INFO' and drop the message without acting on it.
+CVE-2017-14695 Directory traversal vulnerability in minion id validation in SaltStack. Allows remote minions with incorrect credentials to authenticate to a master via a crafted minion ID. Credit for discovering the security flaw goes to: Julian Brost (julian@0x4a42.net)
 
-New master configuration option `drop_messages_signature_fail`
-Drop messages from minions when their signatures do not validate.
-Note that when this option is False but `require_minion_sign_messages` is True
-minions MUST sign their messages but the validity of their signatures
-is ignored.
-
-New minion configuration option `minion_sign_messages`
-Causes the minion to cryptographically sign the payload of messages it places
-on the event bus for the master.  The payloads are signed with the minion's
-private key so the master can verify the signature with its public key.
+CVE-2017-14696 Remote Denial of Service with a specially crafted authentication request. Credit for discovering the security flaw goes to: Julian Brost (julian@0x4a42.net)

doc/topics/releases/2016.3.9.rst

@@ -0,0 +1,29 @@
+===========================
+Salt 2016.3.9 Release Notes
+===========================
+
+Version 2016.3.9 is a bugfix release for :ref:`2016.3.0 <release-2016-3-0>`.
+
+Changes for v2016.3.7..v2016.3.9
+--------------------------------
+
+New master configuration option `allow_minion_key_revoke`, defaults to True.  This option
+controls whether a minion can request that the master revoke its key.  When True, a minion
+can request a key revocation and the master will comply.  If it is False, the key will not
+be revoked by the msater.
+
+New master configuration option `require_minion_sign_messages`
+This requires that minions cryptographically sign the messages they
+publish to the master.  If minions are not signing, then log this information
+at loglevel 'INFO' and drop the message without acting on it.
+
+New master configuration option `drop_messages_signature_fail`
+Drop messages from minions when their signatures do not validate.
+Note that when this option is False but `require_minion_sign_messages` is True
+minions MUST sign their messages but the validity of their signatures
+is ignored.
+
+New minion configuration option `minion_sign_messages`
+Causes the minion to cryptographically sign the payload of messages it places
+on the event bus for the master.  The payloads are signed with the minion's
+private key so the master can verify the signature with its public key.

doc/topics/tutorials/index.rst

@@ -33,3 +33,5 @@ Tutorials Index
 * :ref:`The macOS (Maverick) Developer Step By Step Guide To Salt Installation <tutorial-macos-walk-through>`
 * :ref:`SaltStack Walk-through <tutorial-salt-walk-through>`
 * :ref:`Writing Salt Tests <tutorial-salt-testing>`
+* :ref:`Running Salt States and Commands in Docker Containers <docker-sls>`
+* :ref:`Preseed Minion with Accepted Key <tutorial-preseed-key>`

doc/topics/tutorials/salt_bootstrap.rst

@@ -23,7 +23,7 @@ Supported Operating Systems
 .. note::
 
     In the event you do not see your distribution or version available please
-    review the develop branch on GitHub as it main contain updates that are
+    review the develop branch on GitHub as it may contain updates that are
     not present in the stable release:
     https://github.com/saltstack/salt-bootstrap/tree/develop
 

pkg/rpm/salt-minion

@@ -67,7 +67,7 @@ _su_cmd() {
 
 
 _get_pid() {
-    netstat $NS_NOTRIM -ap --protocol=unix 2>$ERROR_TO_DEVNULL \
+    netstat -n $NS_NOTRIM -ap --protocol=unix 2>$ERROR_TO_DEVNULL \
         | sed -r -e "\|\s${SOCK_DIR}/minion_event_${MINION_ID_HASH}_pub\.ipc$|"'!d; s|/.*||; s/.*\s//;' \
         | uniq
 }
@@ -155,7 +155,7 @@ start() {
             printf "\nPROCESSES:\n" >&2
             ps wwwaxu | grep '[s]alt-minion' >&2
             printf "\nSOCKETS:\n" >&2
-            netstat $NS_NOTRIM -ap --protocol=unix | grep 'salt.*minion' >&2            
+            netstat -n $NS_NOTRIM -ap --protocol=unix | grep 'salt.*minion' >&2
             printf "\nLOG_FILE:\n" >&2
             tail -n 20 "$LOG_FILE" >&2
             printf "\nENVIRONMENT:\n" >&2

salt/auth/ldap.py

@@ -110,6 +110,10 @@ class _LDAPConnection(object):
             self.ldap.set_option(ldap.OPT_REFERRALS, 0)  # Needed for AD
 
             if not anonymous:
+                if self.bindpw is None or len(self.bindpw) < 1:
+                    raise CommandExecutionError(
+                        'LDAP bind password is not set: password cannot be empty if auth.ldap.anonymous is False'
+                    )
                 self.ldap.simple_bind_s(self.binddn, self.bindpw)
         except Exception as ldap_error:
             raise CommandExecutionError(

salt/crypt.py

@@ -606,6 +606,9 @@ class AsyncAuth(object):
                 raise tornado.gen.Return('retry')
             else:
                 raise SaltClientError('Attempt to authenticate with the salt master failed with timeout error')
+        if not isinstance(payload, dict):
+            log.error('Sign-in attempt failed: %s', payload)
+            raise tornado.gen.Return(False)
         if 'load' in payload:
             if 'ret' in payload['load']:
                 if not payload['load']['ret']:

salt/modules/grains.py

@@ -121,7 +121,7 @@ def get(key, default='', delimiter=DEFAULT_TARGET_DELIM, ordered=True):
 
 def has_value(key):
     '''
-    Determine whether a named value exists in the grains dictionary.
+    Determine whether a key exists in the grains dictionary.
 
     Given a grains dictionary that contains the following structure::
 
@@ -137,7 +137,10 @@ def has_value(key):
 
         salt '*' grains.has_value pkg:apache
     '''
-    return True if salt.utils.traverse_dict_and_list(__grains__, key, False) else False
+    return salt.utils.traverse_dict_and_list(
+        __grains__,
+        key,
+        KeyError) is not KeyError
 
 
 def items(sanitize=False):

salt/modules/kmod.py

@@ -123,7 +123,16 @@ def available():
         salt '*' kmod.available
     '''
     ret = []
+
     mod_dir = os.path.join('/lib/modules/', os.uname()[2])
+
+    built_in_file = os.path.join(mod_dir, 'modules.builtin')
+    if os.path.exists(built_in_file):
+        with salt.utils.fopen(built_in_file, 'r') as f:
+            for line in f:
+                # Strip .ko from the basename
+                ret.append(os.path.basename(line)[:-4])
+
     for root, dirs, files in os.walk(mod_dir):
         for fn_ in files:
             if '.ko' in fn_:

salt/netapi/rest_cherrypy/app.py

@@ -474,11 +474,14 @@ logger = logging.getLogger(__name__)
 import cherrypy
 try:
     from cherrypy.lib import cpstats
-except ImportError:
+except AttributeError:
     cpstats = None
     logger.warn('Import of cherrypy.cpstats failed. '
         'Possible upstream bug: '
         'https://github.com/cherrypy/cherrypy/issues/1444')
+except ImportError:
+    cpstats = None
+    logger.warn('Import of cherrypy.cpstats failed.')
 
 import yaml
 import salt.ext.six as six

salt/states/zenoss.py

@@ -6,7 +6,7 @@ State to manage monitoring in Zenoss.
 
 This state module depends on the 'zenoss' Salt execution module.
 
-Allows for setting a state of minions in Zenoss using the Zenoss API. Currently Zenoss 4.x is supported.
+Allows for setting a state of minions in Zenoss using the Zenoss API. Currently Zenoss 4.x and 5.x are supported.
 
 .. code-block:: yaml
 
@@ -30,6 +30,8 @@ def __virtual__():
     '''
     if 'zenoss.add_device' in __salt__:
         return 'zenoss'
+    else:
+        return False, "The zenoss execution module is not available"
 
 
 def monitored(name, device_class=None, collector='localhost', prod_state=None):
@@ -57,21 +59,28 @@ def monitored(name, device_class=None, collector='localhost', prod_state=None):
         ret['comment'] = '{0} is already monitored'.format(name)
 
         # if prod_state is set, ensure it matches with the current state
-        if prod_state:
-            if device['productionState'] != prod_state:
+        if prod_state is not None and device['productionState'] != prod_state:
+            if __opts__['test']:
+                ret['comment'] = '{0} is already monitored but prodState will be updated'.format(name)
+                ret['result'] = None
+            else:
                 __salt__['zenoss.set_prod_state'](prod_state, name)
-                ret['changes'] = {'old': 'prodState == {0}'.format(device['productionState']), 'new': 'prodState == {0}'.format(prod_state)}
-                ret['comment'] = '{0} is already monitored but prodState was incorrect, setting to Production'.format(name)
+                ret['comment'] = '{0} is already monitored but prodState was updated'.format(name)
 
+            ret['changes'] = {
+                'old': 'prodState == {0}'.format(device['productionState']),
+                'new': 'prodState == {0}'.format(prod_state)
+                }
         return ret
 
+    # Device not yet in Zenoss
     if __opts__['test']:
         ret['comment'] = 'The state of "{0}" will be changed.'.format(name)
         ret['changes'] = {'old': 'monitored == False', 'new': 'monitored == True'}
         ret['result'] = None
         return ret
 
-    # Device not yet in Zenoss. Add and check result
+    # Add and check result
     if __salt__['zenoss.add_device'](name, device_class, collector, prod_state):
         ret['result'] = True
         ret['changes'] = {'old': 'monitored == False', 'new': 'monitored == True'}

salt/transport/tcp.py

@@ -623,6 +623,17 @@ class TCPReqServerChannel(salt.transport.mixins.auth.AESReqServerMixin, salt.tra
                     'payload and load must be a dict', header=header))
                 raise tornado.gen.Return()
 
+            try:
+                id_ = payload['load'].get('id', '')
+                if '\0' in id_:
+                    log.error('Payload contains an id with a null byte: %s', payload)
+                    stream.send(self.serial.dumps('bad load: id contains a null byte'))
+                    raise tornado.gen.Return()
+            except TypeError:
+                log.error('Payload contains non-string id: %s', payload)
+                stream.send(self.serial.dumps('bad load: id {0} is not a string'.format(id_)))
+                raise tornado.gen.Return()
+
             # intercept the "_auth" commands, since the main daemon shouldn't know
             # anything about our key auth
             if payload['enc'] == 'clear' and payload.get('load', {}).get('cmd') == '_auth':

salt/transport/zeromq.py

@@ -596,6 +596,17 @@ class ZeroMQReqServerChannel(salt.transport.mixins.auth.AESReqServerMixin, salt.
             stream.send(self.serial.dumps('payload and load must be a dict'))
             raise tornado.gen.Return()
 
+        try:
+            id_ = payload['load'].get('id', '')
+            if '\0' in id_:
+                log.error('Payload contains an id with a null byte: %s', payload)
+                stream.send(self.serial.dumps('bad load: id contains a null byte'))
+                raise tornado.gen.Return()
+        except TypeError:
+            log.error('Payload contains non-string id: %s', payload)
+            stream.send(self.serial.dumps('bad load: id {0} is not a string'.format(id_)))
+            raise tornado.gen.Return()
+
         # intercept the "_auth" commands, since the main daemon shouldn't know
         # anything about our key auth
         if payload['enc'] == 'clear' and payload.get('load', {}).get('cmd') == '_auth':

salt/utils/verify.py

@@ -481,22 +481,15 @@ def clean_path(root, path, subdir=False):
     return ''
 
 
-def clean_id(id_):
-    '''
-    Returns if the passed id is clean.
-    '''
-    if re.search(r'\.\.{sep}'.format(sep=os.sep), id_):
-        return False
-    return True
-
-
 def valid_id(opts, id_):
     '''
     Returns if the passed id is valid
     '''
     try:
-        return bool(clean_path(opts['pki_dir'], id_)) and clean_id(id_)
-    except (AttributeError, KeyError) as e:
+        if any(x in id_ for x in ('/', '\\', '\0')):
+            return False
+        return bool(clean_path(opts['pki_dir'], id_))
+    except (AttributeError, KeyError, TypeError):
         return False
 
 

setup.py

@@ -178,17 +178,22 @@ class WriteSaltVersion(Command):
         '''
 
     def run(self):
-        if not os.path.exists(SALT_VERSION_HARDCODED):
+        if not os.path.exists(SALT_VERSION_HARDCODED) or self.distribution.with_salt_version:
             # Write the version file
             if getattr(self.distribution, 'salt_version_hardcoded_path', None) is None:
                 print('This command is not meant to be called on it\'s own')
                 exit(1)
 
+            if not self.distribution.with_salt_version:
+                salt_version = __saltstack_version__
+            else:
+                salt_version = SaltStackVersion.parse(self.distribution.with_salt_version)
+
             # pylint: disable=E0602
             open(self.distribution.salt_version_hardcoded_path, 'w').write(
                 INSTALL_VERSION_TEMPLATE.format(
                     date=DATE,
-                    full_version_info=__saltstack_version__.full_info
+                    full_version_info=salt_version.full_info
                 )
             )
             # pylint: enable=E0602
@@ -701,6 +706,13 @@ class Build(build):
     def run(self):
         # Run build.run function
         build.run(self)
+        if getattr(self.distribution, 'with_salt_version', False):
+            # Write the hardcoded salt version module salt/_version.py
+            self.distribution.salt_version_hardcoded_path = os.path.join(
+                self.build_lib, 'salt', '_version.py'
+            )
+            self.run_command('write_salt_version')
+
         if getattr(self.distribution, 'running_salt_install', False):
             # If our install attribute is present and set to True, we'll go
             # ahead and write our install time python modules.
@@ -805,6 +817,7 @@ class SaltDistribution(distutils.dist.Distribution):
         ('ssh-packaging', None, 'Run in SSH packaging mode'),
         ('salt-transport=', None, 'The transport to prepare salt for. Choices are \'zeromq\' '
                                   '\'raet\' or \'both\'. Defaults to \'zeromq\'', 'zeromq')] + [
+        ('with-salt-version=', None, 'Set a fixed version for Salt instead calculating it'),
         # Salt's Paths Configuration Settings
         ('salt-root-dir=', None,
          'Salt\'s pre-configured root directory'),
@@ -856,6 +869,9 @@ class SaltDistribution(distutils.dist.Distribution):
         self.salt_spm_pillar_dir = None
         self.salt_spm_reactor_dir = None
 
+        # Salt version
+        self.with_salt_version = None
+
         self.name = 'salt-ssh' if PACKAGED_FOR_SALT_SSH else 'salt'
         self.salt_version = __version__  # pylint: disable=undefined-variable
         self.description = 'Portable, distributed, remote execution and configuration management system'

tests/unit/utils/verify_test.py

@@ -60,6 +60,16 @@ class TestVerify(TestCase):
         opts = {'pki_dir': '/tmp/whatever'}
         self.assertFalse(valid_id(opts, None))
 
+    def test_valid_id_pathsep(self):
+        '''
+        Path separators in id should make it invalid
+        '''
+        opts = {'pki_dir': '/tmp/whatever'}
+        # We have to test both path separators because os.path.normpath will
+        # convert forward slashes to backslashes on Windows.
+        for pathsep in ('/', '\\'):
+            self.assertFalse(valid_id(opts, pathsep.join(('..', 'foobar'))))
+
     def test_zmq_verify(self):
         self.assertTrue(zmq_version())