Merge branch '2016.11' into 'nitrogen'

Conflicts:
  - salt/config/__init__.py
  - salt/modules/cp.py
  - salt/states/saltmod.py
  - salt/utils/__init__.py
  - salt/utils/gzip_util.py
  - tests/integration/shell/test_cp.py

doc/ref/cli/salt-cp.rst

@@ -2,32 +2,42 @@
 ``salt-cp``
 ===========
 
-Copy a file to a set of systems
+Copy a file or files to one or more minions
 
 Synopsis
 ========
 
 .. code-block:: bash
 
-    salt-cp '*' [ options ] SOURCE DEST
+    salt-cp '*' [ options ] SOURCE [SOURCE2 SOURCE3 ...] DEST
 
-    salt-cp -E '.*' [ options ] SOURCE DEST
+    salt-cp -E '.*' [ options ] SOURCE [SOURCE2 SOURCE3 ...] DEST
 
-    salt-cp -G 'os:Arch.*' [ options ] SOURCE DEST
+    salt-cp -G 'os:Arch.*' [ options ] SOURCE [SOURCE2 SOURCE3 ...] DEST
 
 Description
 ===========
 
-Salt copy copies a local file out to all of the Salt minions matched by the
-given target.
+salt-cp copies files from the master to all of the Salt minions matched by the
+specified target expression.
 
-Salt copy is only intended for use with small files (< 100KB). If you need
-to copy large files out to minions please use the cp.get_file function.
+.. note::
+    salt-cp uses Salt's publishing mechanism. This means the privacy of the
+    contents of the file on the wire is completely dependent upon the transport
+    in use. In addition, if the master or minion is running with debug logging,
+    the contents of the file will be logged to disk.
 
-Note: salt-cp uses salt's publishing mechanism. This means the privacy of the
-contents of the file on the wire is completely dependent upon the transport
-in use. In addition, if the salt-master is running with debug logging it is
-possible that the contents of the file will be logged to disk.
+    In addition, this tool is less efficient than the Salt fileserver when
+    copying larger files. It is recommended to instead use
+    :py:func:`cp.get_file <salt.modules.cp.get_file>` to copy larger files to
+    minions. However, this requires the file to be located within one of the
+    fileserver directories.
+
+.. versionchanged:: 2016.3.7,2016.11.6,Nitrogen
+    Compression support added, disable with ``-n``. Also, if the destination
+    path ends in a path separator (i.e. ``/``,  or ``\`` on Windows, the
+    desitination will be assumed to be a directory. Finally, recursion is now
+    supported, allowing for entire directories to be copied.
 
 Options
 =======
@@ -46,6 +56,12 @@ Options
 .. include:: _includes/target-selection.rst
 
 
+.. option:: -n, --no-compression
+
+    Disable gzip compression.
+
+    .. versionadded:: 2016.3.7,2016.11.6,Nitrogen
+
 See also
 ========
 

doc/ref/configuration/master.rst

@@ -1048,7 +1048,8 @@ This is completely disabled by default.
         - root
         - '^(?!sudo_).*$'   #  all non sudo users
       modules:
-        - cmd
+        - cmd.*
+        - test.echo
 
 .. conf_master:: external_auth
 

salt/cli/cp.py

@@ -9,15 +9,26 @@ Salt-cp can be used to distribute configuration files
 # Import python libs
 from __future__ import print_function
 from __future__ import absolute_import
+import base64
+import errno
+import logging
 import os
+import re
 import sys
 
 # Import salt libs
 import salt.client
-from salt.utils import parsers, print_cli
+import salt.utils.gzip_util
+import salt.utils.minions
+from salt.utils import parsers, to_bytes
 from salt.utils.verify import verify_log
 import salt.output
 
+# Import 3rd party libs
+from salt.ext import six
+
+log = logging.getLogger(__name__)
+
 
 class SaltCPCli(parsers.SaltCPOptionParser):
     '''
@@ -44,65 +55,168 @@ class SaltCP(object):
     '''
     def __init__(self, opts):
         self.opts = opts
+        self.is_windows = salt.utils.is_windows()
 
-    def _file_dict(self, fn_):
-        '''
-        Take a path and return the contents of the file as a string
-        '''
-        if not os.path.isfile(fn_):
-            err = 'The referenced file, {0} is not available.'.format(fn_)
-            sys.stderr.write(err + '\n')
-            sys.exit(42)
-        with salt.utils.fopen(fn_, 'r') as fp_:
-            data = fp_.read()
-        return {fn_: data}
+    def _mode(self, path):
+        if self.is_windows:
+            return None
+        try:
+            return int(oct(os.stat(path).st_mode)[-4:], 8)
+        except (TypeError, IndexError, ValueError):
+            return None
 
-    def _recurse_dir(self, fn_, files=None):
+    def _recurse(self, path):
         '''
-        Recursively pull files from a directory
-        '''
-        if files is None:
-            files = {}
-
-        for base in os.listdir(fn_):
-            path = os.path.join(fn_, base)
-            if os.path.isdir(path):
-                files.update(self._recurse_dir(path))
-            else:
-                files.update(self._file_dict(path))
-        return files
-
-    def _load_files(self):
-        '''
-        Parse the files indicated in opts['src'] and load them into a python
-        object for transport
+        Get a list of all specified files
         '''
         files = {}
+        empty_dirs = []
+        try:
+            sub_paths = os.listdir(path)
+        except OSError as exc:
+            if exc.errno == errno.ENOENT:
+                # Path does not exist
+                sys.stderr.write('{0} does not exist\n'.format(path))
+                sys.exit(42)
+            elif exc.errno in (errno.EINVAL, errno.ENOTDIR):
+                # Path is a file (EINVAL on Windows, ENOTDIR otherwise)
+                files[path] = self._mode(path)
+        else:
+            if not sub_paths:
+                empty_dirs.append(path)
+            for fn_ in sub_paths:
+                files_, empty_dirs_ = self._recurse(os.path.join(path, fn_))
+                files.update(files_)
+                empty_dirs.extend(empty_dirs_)
+
+        return files, empty_dirs
+
+    def _list_files(self):
+        files = {}
+        empty_dirs = set()
         for fn_ in self.opts['src']:
-            if os.path.isfile(fn_):
-                files.update(self._file_dict(fn_))
-            elif os.path.isdir(fn_):
-                print_cli(fn_ + ' is a directory, only files are supported.')
-                #files.update(self._recurse_dir(fn_))
-        return files
+            files_, empty_dirs_ = self._recurse(fn_)
+            files.update(files_)
+            empty_dirs.update(empty_dirs_)
+        return files, sorted(empty_dirs)
 
     def run(self):
         '''
         Make the salt client call
         '''
-        arg = [self._load_files(), self.opts['dest']]
+        files, empty_dirs = self._list_files()
+        dest = self.opts['dest']
+        gzip = self.opts['gzip']
+        tgt = self.opts['tgt']
+        timeout = self.opts['timeout']
+        selected_target_option = self.opts.get('selected_target_option')
+
+        dest_is_dir = bool(empty_dirs) \
+            or len(files) > 1 \
+            or bool(re.search(r'[\\/]$', dest))
+
+        reader = salt.utils.gzip_util.compress_file \
+            if gzip \
+            else salt.utils.itertools.read_file
+
+        minions = salt.utils.minions.CkMinions(self.opts).check_minions(
+            tgt,
+            expr_form=selected_target_option or 'glob')
+
         local = salt.client.get_local_client(self.opts['conf_file'])
-        args = [self.opts['tgt'],
-                'cp.recv',
-                arg,
-                self.opts['timeout'],
+
+        def _get_remote_path(fn_):
+            if fn_ in self.opts['src']:
+                # This was a filename explicitly passed on the CLI
+                return os.path.join(dest, os.path.basename(fn_)) \
+                    if dest_is_dir \
+                    else dest
+            else:
+                for path in self.opts['src']:
+                    relpath = os.path.relpath(fn_, path + os.sep)
+                    if relpath.startswith(parent):
+                        # File is not within this dir
+                        continue
+                    return os.path.join(dest, os.path.basename(path), relpath)
+                else:  # pylint: disable=useless-else-on-loop
+                    # Should not happen
+                    log.error('Failed to find remote path for %s', fn_)
+                    return None
+
+        ret = {}
+        parent = '..' + os.sep
+        for fn_, mode in six.iteritems(files):
+            remote_path = _get_remote_path(fn_)
+
+            index = 1
+            failed = {}
+            for chunk in reader(fn_, chunk_size=self.opts['salt_cp_chunk_size']):
+                chunk = base64.b64encode(to_bytes(chunk))
+                append = index > 1
+                log.debug(
+                    'Copying %s to %starget \'%s\' as %s%s',
+                    fn_,
+                    '{0} '.format(selected_target_option)
+                        if selected_target_option
+                        else '',
+                    tgt,
+                    remote_path,
+                    ' (chunk #{0})'.format(index) if append else ''
+                )
+                args = [
+                    tgt,
+                    'cp.recv',
+                    [remote_path, chunk, append, gzip, mode],
+                    timeout,
                 ]
+                if selected_target_option is not None:
+                    args.append(selected_target_option)
 
-        selected_target_option = self.opts.get('selected_target_option', None)
-        if selected_target_option is not None:
-            args.append(selected_target_option)
+                result = local.cmd(*args)
 
-        ret = local.cmd(*args)
+                if not result:
+                    # Publish failed
+                    msg = (
+                        'Publish failed.{0} It may be necessary to '
+                        'decrease salt_cp_chunk_size (current value: '
+                        '{1})'.format(
+                            ' File partially transferred.' if index > 1 else '',
+                            self.opts['salt_cp_chunk_size'],
+                        )
+                    )
+                    for minion in minions:
+                        ret.setdefault(minion, {})[remote_path] = msg
+                    break
+
+                for minion_id, minion_ret in six.iteritems(result):
+                    ret.setdefault(minion_id, {})[remote_path] = minion_ret
+                    # Catch first error message for a given minion, we will
+                    # rewrite the results after we're done iterating through
+                    # the chunks.
+                    if minion_ret is not True and minion_id not in failed:
+                        failed[minion_id] = minion_ret
+
+                index += 1
+
+            for minion_id, msg in six.iteritems(failed):
+                ret[minion_id][remote_path] = msg
+
+        for dirname in empty_dirs:
+            remote_path = _get_remote_path(dirname)
+            log.debug(
+                'Creating empty dir %s on %starget \'%s\'',
+                dirname,
+                '{0} '.format(selected_target_option)
+                    if selected_target_option
+                    else '',
+                tgt,
+            )
+            args = [tgt, 'cp.recv', [remote_path, None], timeout]
+            if selected_target_option is not None:
+                args.append(selected_target_option)
+
+            for minion_id, minion_ret in six.iteritems(local.cmd(*args)):
+                ret.setdefault(minion_id, {})[remote_path] = minion_ret
 
         salt.output.display_output(
                 ret,

salt/client/__init__.py

@@ -1246,8 +1246,17 @@ class LocalClient(object):
                         minions.remove(raw['data']['id'])
                         break
                 except KeyError as exc:
-                    # This is a safe pass. We're just using the try/except to avoid having to deep-check for keys
-                    log.debug('Passing on saltutil error. This may be an error in saltclient. {0}'.format(exc))
+                    # This is a safe pass. We're just using the try/except to
+                    # avoid having to deep-check for keys.
+                    missing_key = exc.__str__().strip('\'"')
+                    if missing_key == 'retcode':
+                        log.debug('retcode missing from client return')
+                    else:
+                        log.debug(
+                            'Passing on saltutil error. Key \'%s\' missing '
+                            'from client return. This may be an error in '
+                            'the client.', missing_key
+                        )
                 # Keep track of the jid events to unsubscribe from later
                 open_jids.add(jinfo['jid'])
 

salt/config/__init__.py

@@ -1042,6 +1042,9 @@ VALID_OPTS = {
 
     # Permit or deny allowing minions to request revoke of its own key
     'allow_minion_key_revoke': bool,
+
+    # File chunk size for salt-cp
+    'salt_cp_chunk_size': int,
 }
 
 # default configurations
@@ -1302,6 +1305,7 @@ DEFAULT_MINION_OPTS = {
     'beacons_before_connect': False,
     'scheduler_before_connect': False,
     'cache': 'localfs',
+    'salt_cp_chunk_size': 65536,
     'extmod_whitelist': {},
     'extmod_blacklist': {},
 }
@@ -1598,6 +1602,7 @@ DEFAULT_MASTER_OPTS = {
     'django_auth_path': '',
     'django_auth_settings': '',
     'allow_minion_key_revoke': True,
+    'salt_cp_chunk_size': 98304,
 }
 
 

salt/modules/cp.py

@@ -5,6 +5,8 @@ Minion side functions for salt-cp
 
 # Import python libs
 from __future__ import absolute_import
+import base64
+import errno
 import os
 import logging
 import fnmatch
@@ -14,6 +16,7 @@ import salt.minion
 import salt.fileclient
 import salt.utils
 import salt.utils.files
+import salt.utils.gzip_util
 import salt.utils.url
 import salt.crypt
 import salt.transport
@@ -55,33 +58,69 @@ def _gather_pillar(pillarenv, pillar_override):
     return ret
 
 
-def recv(files, dest):
+def recv(dest, chunk, append=False, compressed=True, mode=None):
     '''
-    Used with salt-cp, pass the files dict, and the destination.
-
-    This function receives small fast copy files from the master via salt-cp.
-    It does not work via the CLI.
+    This function receives files copied to the minion using ``salt-cp`` and is
+    not intended to be used directly on the CLI.
     '''
-    ret = {}
-    for path, data in six.iteritems(files):
-        if os.path.basename(path) == os.path.basename(dest) \
-                and not os.path.isdir(dest):
-            final = dest
-        elif os.path.isdir(dest):
-            final = os.path.join(dest, os.path.basename(path))
-        elif os.path.isdir(os.path.dirname(dest)):
-            final = dest
-        else:
-            return 'Destination unavailable'
+    if 'retcode' not in __context__:
+        __context__['retcode'] = 0
 
+    def _error(msg):
+        __context__['retcode'] = 1
+        return msg
+
+    if chunk is None:
+        # dest is an empty dir and needs to be created
         try:
-            with salt.utils.fopen(final, 'w+') as fp_:
-                fp_.write(data)
-            ret[final] = True
-        except IOError:
-            ret[final] = False
+            os.makedirs(dest)
+        except OSError as exc:
+            if exc.errno == errno.EEXIST:
+                if os.path.isfile(dest):
+                    return 'Path exists and is a file'
+            else:
+                return _error(exc.__str__())
+        return True
 
-    return ret
+    chunk = base64.b64decode(chunk)
+
+    open_mode = 'ab' if append else 'wb'
+    try:
+        fh_ = salt.utils.fopen(dest, open_mode)
+    except (IOError, OSError) as exc:
+        if exc.errno != errno.ENOENT:
+            # Parent dir does not exist, we need to create it
+            return _error(exc.__str__())
+        try:
+            os.makedirs(os.path.dirname(dest))
+        except (IOError, OSError) as makedirs_exc:
+            # Failed to make directory
+            return _error(makedirs_exc.__str__())
+        fh_ = salt.utils.fopen(dest, open_mode)
+
+    try:
+        # Write the chunk to disk
+        fh_.write(salt.utils.gzip_util.uncompress(chunk) if compressed
+                  else chunk)
+    except (IOError, OSError) as exc:
+        # Write failed
+        return _error(exc.__str__())
+    else:
+        # Write successful
+        if not append and mode is not None:
+            # If this is the first chunk we're writing, set the mode
+            #log.debug('Setting mode for %s to %s', dest, oct(mode))
+            log.debug('Setting mode for %s to %s', dest, mode)
+            try:
+                os.chmod(dest, mode)
+            except OSError:
+                return _error(exc.__str__())
+        return True
+    finally:
+        try:
+            fh_.close()
+        except AttributeError:
+            pass
 
 
 def _mk_client():

salt/modules/napalm_network.py

@@ -148,6 +148,11 @@ def _config_logic(napalm_device,
     if _compare.get('result', False):
         loaded_result['diff'] = _compare.get('out')
         loaded_result.pop('out', '')  # not needed
+    else:
+        loaded_result['diff'] = None
+        loaded_result['result'] = False
+        loaded_result['comment'] = _compare.get('comment')
+        return loaded_result
 
     _loaded_res = loaded_result.get('result', False)
     if not _loaded_res or test:

salt/modules/network.py

@@ -1166,7 +1166,10 @@ def mod_hostname(hostname):
         with salt.utils.fopen('/etc/sysconfig/network', 'w') as fh_:
             for net in network_c:
                 if net.startswith('HOSTNAME'):
-                    fh_.write('HOSTNAME={0}\n'.format(hostname))
+                    old_hostname = net.split('=', 1)[1].rstrip()
+                    quote_type = salt.utils.is_quoted(old_hostname)
+                    fh_.write('HOSTNAME={1}{0}{1}\n'.format(
+                        salt.utils.dequote(hostname), quote_type))
                 else:
                     fh_.write(net)
     elif __grains__['os_family'] in ('Debian', 'NILinuxRT'):

salt/modules/neutron.py

@@ -39,6 +39,30 @@ Module for handling OpenStack Neutron calls
     For example::
 
         salt '*' neutron.network_list profile=openstack1
+
+    To use keystoneauth1 instead of keystoneclient, include the `use_keystoneauth`
+    option in the pillar or minion config.
+
+    .. note:: this is required to use keystone v3 as for authentication.
+
+    .. code-block:: yaml
+
+        keystone.user: admin
+        keystone.password: verybadpass
+        keystone.tenant: admin
+        keystone.auth_url: 'http://127.0.0.1:5000/v3/'
+        keystone.region_name: 'RegionOne'
+        keystone.service_type: 'network'
+        keystone.use_keystoneauth: true
+        keystone.verify: '/path/to/custom/certs/ca-bundle.crt'
+
+
+    Note: by default the neutron module will attempt to verify its connection
+    utilizing the system certificates. If you need to verify against another bundle
+    of CA certificates or want to skip verification altogether you will need to
+    specify the `verify` option. You can specify True or False to verify (or not)
+    against system certificates, a path to a bundle or CA certs to check against, or
+    None to allow keystoneauth to search for the certificates on its own.(defaults to True)
 '''
 
 # Import python libs
@@ -83,7 +107,10 @@ def _auth(profile=None):
         tenant = credentials['keystone.tenant']
         auth_url = credentials['keystone.auth_url']
         region_name = credentials.get('keystone.region_name', None)
-        service_type = credentials['keystone.service_type']
+        service_type = credentials.get('keystone.service_type', 'network')
+        os_auth_system = credentials.get('keystone.os_auth_system', None)
+        use_keystoneauth = credentials.get('keystone.use_keystoneauth', False)
+        verify = credentials.get('keystone.verify', True)
     else:
         user = __salt__['config.option']('keystone.user')
         password = __salt__['config.option']('keystone.password')
@@ -91,15 +118,37 @@ def _auth(profile=None):
         auth_url = __salt__['config.option']('keystone.auth_url')
         region_name = __salt__['config.option']('keystone.region_name')
         service_type = __salt__['config.option']('keystone.service_type')
+        os_auth_system = __salt__['config.option']('keystone.os_auth_system')
+        use_keystoneauth = __salt__['config.option']('keystone.use_keystoneauth')
+        verify = __salt__['config.option']('keystone.verify')
 
-    kwargs = {
-        'username': user,
-        'password': password,
-        'tenant_name': tenant,
-        'auth_url': auth_url,
-        'region_name': region_name,
-        'service_type': service_type
-    }
+    if use_keystoneauth is True:
+        project_domain_name = credentials['keystone.project_domain_name']
+        user_domain_name = credentials['keystone.user_domain_name']
+
+        kwargs = {
+            'username': user,
+            'password': password,
+            'tenant_name': tenant,
+            'auth_url': auth_url,
+            'region_name': region_name,
+            'service_type': service_type,
+            'os_auth_plugin': os_auth_system,
+            'use_keystoneauth': use_keystoneauth,
+            'verify': verify,
+            'project_domain_name': project_domain_name,
+            'user_domain_name': user_domain_name
+        }
+    else:
+        kwargs = {
+            'username': user,
+            'password': password,
+            'tenant_name': tenant,
+            'auth_url': auth_url,
+            'region_name': region_name,
+            'service_type': service_type,
+            'os_auth_plugin': os_auth_system
+        }
 
     return suoneu.SaltNeutron(**kwargs)
 

salt/modules/nova.py

@@ -35,15 +35,34 @@ Module for handling OpenStack Nova calls
     For example::
 
         salt '*' nova.flavor_list profile=openstack1
+
+    To use keystoneauth1 instead of keystoneclient, include the `use_keystoneauth`
+    option in the pillar or minion config.
+
+    .. note:: this is required to use keystone v3 as for authentication.
+
+    .. code-block:: yaml
+
+        keystone.user: admin
+        keystone.password: verybadpass
+        keystone.tenant: admin
+        keystone.auth_url: 'http://127.0.0.1:5000/v3/'
+        keystone.use_keystoneauth: true
+        keystone.verify: '/path/to/custom/certs/ca-bundle.crt'
+
+
+    Note: by default the nova module will attempt to verify its connection
+    utilizing the system certificates. If you need to verify against another bundle
+    of CA certificates or want to skip verification altogether you will need to
+    specify the `verify` option. You can specify True or False to verify (or not)
+    against system certificates, a path to a bundle or CA certs to check against, or
+    None to allow keystoneauth to search for the certificates on its own.(defaults to True)
 '''
 from __future__ import absolute_import
 
 # Import python libs
 import logging
 
-# Import salt libs
-import salt.utils.openstack.nova as suon
-
 
 # Get logging started
 log = logging.getLogger(__name__)
@@ -53,8 +72,11 @@ __func_alias__ = {
     'list_': 'list'
 }
 
-# Define the module's virtual name
-__virtualname__ = 'nova'
+try:
+    import salt.utils.openstack.nova as suon
+    HAS_NOVA = True
+except NameError as exc:
+    HAS_NOVA = False
 
 
 def __virtual__():
@@ -62,10 +84,7 @@ def __virtual__():
     Only load this module if nova
     is installed on this minion.
     '''
-    if suon.check_nova():
-        return __virtualname__
-    return (False, 'The nova execution module failed to load: '
-            'only available if nova is installed.')
+    return HAS_NOVA
 
 
 __opts__ = {}
@@ -84,6 +103,8 @@ def _auth(profile=None):
         region_name = credentials.get('keystone.region_name', None)
         api_key = credentials.get('keystone.api_key', None)
         os_auth_system = credentials.get('keystone.os_auth_system', None)
+        use_keystoneauth = credentials.get('keystone.use_keystoneauth', False)
+        verify = credentials.get('keystone.verify', None)
     else:
         user = __salt__['config.option']('keystone.user')
         password = __salt__['config.option']('keystone.password')
@@ -92,15 +113,34 @@ def _auth(profile=None):
         region_name = __salt__['config.option']('keystone.region_name')
         api_key = __salt__['config.option']('keystone.api_key')
         os_auth_system = __salt__['config.option']('keystone.os_auth_system')
-    kwargs = {
-        'username': user,
-        'password': password,
-        'api_key': api_key,
-        'project_id': tenant,
-        'auth_url': auth_url,
-        'region_name': region_name,
-        'os_auth_plugin': os_auth_system
-    }
+        use_keystoneauth = __salt__['config.option']('keystone.use_keystoneauth')
+        verify = __salt__['config.option']('keystone.verify')
+
+    if use_keystoneauth is True:
+        project_domain_name = credentials['keystone.project_domain_name']
+        user_domain_name = credentials['keystone.user_domain_name']
+
+        kwargs = {
+            'username': user,
+            'password': password,
+            'project_id': tenant,
+            'auth_url': auth_url,
+            'region_name': region_name,
+            'use_keystoneauth': use_keystoneauth,
+            'verify': verify,
+            'project_domain_name': project_domain_name,
+            'user_domain_name': user_domain_name
+        }
+    else:
+        kwargs = {
+            'username': user,
+            'password': password,
+            'api_key': api_key,
+            'project_id': tenant,
+            'auth_url': auth_url,
+            'region_name': region_name,
+            'os_auth_plugin': os_auth_system
+        }
 
     return suon.SaltNova(**kwargs)
 

salt/modules/rh_ip.py

@@ -799,21 +799,30 @@ def _parse_network_settings(opts, current):
     retain_settings = opts.get('retain_settings', False)
     result = current if retain_settings else {}
 
+    # Default quote type is an empty string, which will not quote values
+    quote_type = ''
+
     valid = _CONFIG_TRUE + _CONFIG_FALSE
     if 'enabled' not in opts:
         try:
             opts['networking'] = current['networking']
+            # If networking option is quoted, use its quote type
+            quote_type = salt.utils.is_quoted(opts['networking'])
             _log_default_network('networking', current['networking'])
         except ValueError:
             _raise_error_network('networking', valid)
     else:
         opts['networking'] = opts['enabled']
 
-    if opts['networking'] in valid:
-        if opts['networking'] in _CONFIG_TRUE:
-            result['networking'] = 'yes'
-        elif opts['networking'] in _CONFIG_FALSE:
-            result['networking'] = 'no'
+    true_val = '{0}yes{0}'.format(quote_type)
+    false_val = '{0}no{0}'.format(quote_type)
+
+    networking = salt.utils.dequote(opts['networking'])
+    if networking in valid:
+        if networking in _CONFIG_TRUE:
+            result['networking'] = true_val
+        elif networking in _CONFIG_FALSE:
+            result['networking'] = false_val
     else:
         _raise_error_network('networking', valid)
 
@@ -825,22 +834,25 @@ def _parse_network_settings(opts, current):
             _raise_error_network('hostname', ['server1.example.com'])
 
     if opts['hostname']:
-        result['hostname'] = opts['hostname']
+        result['hostname'] = '{1}{0}{1}'.format(
+            salt.utils.dequote(opts['hostname']), quote_type)
     else:
         _raise_error_network('hostname', ['server1.example.com'])
 
     if 'nozeroconf' in opts:
-        if opts['nozeroconf'] in valid:
-            if opts['nozeroconf'] in _CONFIG_TRUE:
-                result['nozeroconf'] = 'true'
-            elif opts['nozeroconf'] in _CONFIG_FALSE:
-                result['nozeroconf'] = 'false'
+        nozeroconf = salt.utils.dequote(opts['nozerconf'])
+        if nozeroconf in valid:
+            if nozeroconf in _CONFIG_TRUE:
+                result['nozeroconf'] = true_val
+            elif nozeroconf in _CONFIG_FALSE:
+                result['nozeroconf'] = false_val
         else:
             _raise_error_network('nozeroconf', valid)
 
     for opt in opts:
         if opt not in ['networking', 'hostname', 'nozeroconf']:
-            result[opt] = opts[opt]
+            result[opt] = '{1}{0}{1}'.format(
+                salt.utils.dequote(opts[opt]), quote_type)
     return result
 
 

salt/modules/useradd.py

@@ -436,7 +436,7 @@ def chgroups(name, groups, append=False, root=None):
         if result['retcode'] != 0 and 'not found in' in result['stderr']:
             ret = True
             for group in groups:
-                cmd = ['gpasswd', '-a', '{0}'.format(name), '{1}'.format(group)]
+                cmd = ['gpasswd', '-a', '{0}'.format(name), '{0}'.format(group)]
                 if __salt__['cmd.retcode'](cmd, python_shell=False) != 0:
                     ret = False
             return ret

salt/modules/yumpkg.py

@@ -471,10 +471,11 @@ def latest_version(*names, **kwargs):
     def _check_cur(pkg):
         if pkg.name in cur_pkgs:
             for installed_version in cur_pkgs[pkg.name]:
-                # If any installed version is greater than the one found by
-                # yum/dnf list available, then it is not an upgrade.
+                # If any installed version is greater than (or equal to) the
+                # one found by yum/dnf list available, then it is not an
+                # upgrade.
                 if salt.utils.compare_versions(ver1=installed_version,
-                                               oper='>',
+                                               oper='>=',
                                                ver2=pkg.version,
                                                cmp_func=version_cmp):
                     return False

salt/states/saltmod.py

@@ -252,7 +252,7 @@ def state(name,
     elif __opts__.get('pillarenv'):
         cmd_kw['kwarg']['pillarenv'] = __opts__['pillarenv']
 
-    cmd_kw['kwarg']['saltenv'] = saltenv
+    cmd_kw['kwarg']['saltenv'] = saltenv if saltenv is not None else __env__
     cmd_kw['kwarg']['queue'] = queue
 
     if isinstance(concurrent, bool):

salt/transport/ipc.py

@@ -177,13 +177,25 @@ class IPCServer(object):
                     body = framed_msg['body']
                     self.io_loop.spawn_callback(self.payload_handler, body, write_callback(stream, framed_msg['head']))
             except tornado.iostream.StreamClosedError:
-                log.trace('Client disconnected from IPC {0}'.format(self.socket_path))
+                log.trace('Client disconnected '
+                          'from IPC {0}'.format(self.socket_path))
                 break
+            except socket.error as exc:
+                # On occasion an exception will occur with
+                # an error code of 0, it's a spurious exception.
+                if exc.errno == 0:
+                    log.trace('Exception occured with error number 0, '
+                              'spurious exception: {0}'.format(exc))
+                else:
+                    log.error('Exception occurred while '
+                              'handling stream: {0}'.format(exc))
             except Exception as exc:
-                log.error('Exception occurred while handling stream: {0}'.format(exc))
+                log.error('Exception occurred while '
+                          'handling stream: {0}'.format(exc))
 
     def handle_connection(self, connection, address):
-        log.trace('IPCServer: Handling connection to address: {0}'.format(address))
+        log.trace('IPCServer: Handling connection '
+                  'to address: {0}'.format(address))
         try:
             stream = IOStream(
                 connection,

salt/utils/__init__.py

@@ -3411,3 +3411,26 @@ def fnmatch_multiple(candidates, pattern):
         except TypeError:
             pass
     return None
+
+
+def is_quoted(val):
+    '''
+    Return a single or double quote, if a string is wrapped in extra quotes.
+    Otherwise return an empty string.
+    '''
+    ret = ''
+    if (
+        isinstance(val, six.string_types) and val[0] == val[-1] and
+        val.startswith(('\'', '"'))
+    ):
+        ret = val[0]
+    return ret
+
+
+def dequote(val):
+    '''
+    Remove extra quotes around a string.
+    '''
+    if is_quoted(val):
+        return val[1:-1]
+    return val

salt/utils/gzip_util.py

@@ -10,9 +10,12 @@ from __future__ import absolute_import
 # Import python libs
 import gzip
 
+# Import Salt libs
+import salt.utils
+
 # Import 3rd-party libs
 import salt.ext.six as six
-from salt.ext.six import BytesIO
+from salt.ext.six import BytesIO, StringIO
 
 
 class GzipFile(gzip.GzipFile):
@@ -66,3 +69,38 @@ def uncompress(data):
     with open_fileobj(buf, 'rb') as igz:
         unc = igz.read()
         return unc
+
+
+def compress_file(fh_, compresslevel=9, chunk_size=1048576):
+    '''
+    Generator that reads chunk_size bytes at a time from a file/filehandle and
+    yields the compressed result of each read.
+
+    .. note::
+        Each chunk is compressed separately. They cannot be stitched together
+        to form a compressed file. This function is designed to break up a file
+        into compressed chunks for transport and decompression/reassembly on a
+        remote host.
+    '''
+    try:
+        bytes_read = int(chunk_size)
+        if bytes_read != chunk_size:
+            raise ValueError
+    except ValueError:
+        raise ValueError('chunk_size must be an integer')
+    try:
+        while bytes_read == chunk_size:
+            buf = StringIO()
+            with open_fileobj(buf, 'wb', compresslevel) as ogz:
+                try:
+                    bytes_read = ogz.write(fh_.read(chunk_size))
+                except AttributeError:
+                    # Open the file and re-attempt the read
+                    fh_ = salt.utils.fopen(fh_, 'rb')
+                    bytes_read = ogz.write(fh_.read(chunk_size))
+            yield buf.getvalue()
+    finally:
+        try:
+            fh_.close()
+        except AttributeError:
+            pass

salt/utils/itertools.py

@@ -7,6 +7,9 @@ Helpful generators and other tools
 from __future__ import absolute_import
 import re
 
+# Import Salt libs
+import salt.utils
+
 
 def split(orig, sep=None):
     '''
@@ -32,3 +35,31 @@ def split(orig, sep=None):
         if pos < match.start() or sep is not None:
             yield orig[pos:match.start()]
         pos = match.end()
+
+
+def read_file(fh_, chunk_size=1048576):
+    '''
+    Generator that reads chunk_size bytes at a time from a file/filehandle and
+    yields it.
+    '''
+    try:
+        if chunk_size != int(chunk_size):
+            raise ValueError
+    except ValueError:
+        raise ValueError('chunk_size must be an integer')
+    try:
+        while True:
+            try:
+                chunk = fh_.read(chunk_size)
+            except AttributeError:
+                # Open the file and re-attempt the read
+                fh_ = salt.utils.fopen(fh_, 'rb')
+                chunk = fh_.read(chunk_size)
+            if not chunk:
+                break
+            yield chunk
+    finally:
+        try:
+            fh_.close()
+        except AttributeError:
+            pass

salt/utils/openstack/neutron.py

@@ -19,6 +19,14 @@ try:
     HAS_NEUTRON = True
 except ImportError:
     pass
+
+HAS_KEYSTONEAUTH = False
+try:
+    import keystoneauth1.loading
+    import keystoneauth1.session
+    HAS_KEYSTONEAUTH = True
+except ImportError:
+    pass
 # pylint: enable=import-error
 
 # Import salt libs
@@ -32,11 +40,15 @@ def check_neutron():
     return HAS_NEUTRON
 
 
+def check_keystone():
+    return HAS_KEYSTONEAUTH
+
+
 def sanitize_neutronclient(kwargs):
     variables = (
         'username', 'user_id', 'password', 'token', 'tenant_name',
         'tenant_id', 'auth_url', 'service_type', 'endpoint_type',
-        'region_name', 'endpoint_url', 'timeout', 'insecure',
+        'region_name', 'verify', 'endpoint_url', 'timeout', 'insecure',
         'ca_cert', 'retries', 'raise_error', 'session', 'auth'
     )
     ret = {}
@@ -53,14 +65,70 @@ class SaltNeutron(NeutronShell):
     Class for all neutronclient functions
     '''
 
-    def __init__(self, username, tenant_name, auth_url, password=None,
-                 region_name=None, service_type=None, **kwargs):
+    def __init__(
+        self,
+        username,
+        tenant_name,
+        auth_url,
+        password=None,
+        region_name=None,
+        service_type='network',
+        os_auth_plugin=None,
+        use_keystoneauth=False,
+        **kwargs
+    ):
+
         '''
         Set up neutron credentials
         '''
         if not HAS_NEUTRON:
             return None
 
+        elif all([use_keystoneauth, HAS_KEYSTONEAUTH]):
+            self._new_init(username=username,
+                           project_name=tenant_name,
+                           auth_url=auth_url,
+                           region_name=region_name,
+                           service_type=service_type,
+                           os_auth_plugin=os_auth_plugin,
+                           password=password,
+                           **kwargs)
+        else:
+            self._old_init(username=username,
+                           tenant_name=tenant_name,
+                           auth_url=auth_url,
+                           region_name=region_name,
+                           service_type=service_type,
+                           os_auth_plugin=os_auth_plugin,
+                           password=password,
+                           **kwargs)
+
+    def _new_init(self, username, project_name, auth_url, region_name, service_type, password, os_auth_plugin, auth=None, verify=True, **kwargs):
+        if auth is None:
+            auth = {}
+
+        loader = keystoneauth1.loading.get_plugin_loader(os_auth_plugin or 'password')
+
+        self.client_kwargs = kwargs.copy()
+        self.kwargs = auth.copy()
+
+        self.kwargs['username'] = username
+        self.kwargs['project_name'] = project_name
+        self.kwargs['auth_url'] = auth_url
+        self.kwargs['password'] = password
+        if auth_url.endswith('3'):
+            self.kwargs['user_domain_name'] = kwargs.get('user_domain_name', 'default')
+            self.kwargs['project_domain_name'] = kwargs.get('project_domain_name', 'default')
+
+        self.client_kwargs['region_name'] = region_name
+        self.client_kwargs['service_type'] = service_type
+
+        self.client_kwargs = sanitize_neutronclient(self.client_kwargs)
+        options = loader.load_from_options(**self.kwargs)
+        self.session = keystoneauth1.session.Session(auth=options, verify=verify)
+        self.network_conn = client.Client(session=self.session, **self.client_kwargs)
+
+    def _old_init(self, username, tenant_name, auth_url, region_name, service_type, password, os_auth_plugin, auth=None, verify=True, **kwargs):
         self.kwargs = kwargs.copy()
 
         self.kwargs['username'] = username
@@ -69,6 +137,7 @@ class SaltNeutron(NeutronShell):
         self.kwargs['service_type'] = service_type
         self.kwargs['password'] = password
         self.kwargs['region_name'] = region_name
+        self.kwargs['verify'] = verify
 
         self.kwargs = sanitize_neutronclient(self.kwargs)
 

salt/utils/parsers.py

@@ -2127,6 +2127,17 @@ class SaltCPOptionParser(six.with_metaclass(OptionParserMeta,
     _default_logging_level_ = config.DEFAULT_MASTER_OPTS['log_level']
     _default_logging_logfile_ = config.DEFAULT_MASTER_OPTS['log_file']
 
+    def _mixin_setup(self):
+        file_opts_group = optparse.OptionGroup(self, 'File Options')
+        file_opts_group.add_option(
+            '-n', '--no-compression',
+            default=True,
+            dest='compression',
+            action='store_false',
+            help='Disable gzip compression.'
+        )
+        self.add_option_group(file_opts_group)
+
     def _mixin_after_parsed(self):
         # salt-cp needs arguments
         if len(self.args) <= 1:
@@ -2140,8 +2151,9 @@ class SaltCPOptionParser(six.with_metaclass(OptionParserMeta,
                 self.config['tgt'] = self.args[0].split()
         else:
             self.config['tgt'] = self.args[0]
-        self.config['src'] = self.args[1:-1]
+        self.config['src'] = [os.path.realpath(x) for x in self.args[1:-1]]
         self.config['dest'] = self.args[-1]
+        self.config['gzip'] = True
 
     def setup_config(self):
         return config.master_config(self.get_config_file_path())

tests/integration/shell/test_cp.py

@@ -9,9 +9,11 @@
 
 # Import python libs
 from __future__ import absolute_import
+import errno
 import os
 import pipes
 import shutil
+import tempfile
 
 # Import 3rd-party libs
 import yaml
@@ -114,18 +116,13 @@ class CopyTest(ShellCase, ShellCaseCommonTestsMixin):
             self.assertTrue(data[minion])
 
     def test_issue_7754(self):
-        try:
-            old_cwd = os.getcwd()
-        except OSError:
-            # Jenkins throws an OSError from os.getcwd()??? Let's not worry
-            # about it
-            old_cwd = None
-
         config_dir = os.path.join(TMP, 'issue-7754')
-        if not os.path.isdir(config_dir):
-            os.makedirs(config_dir)
 
-        os.chdir(config_dir)
+        try:
+            os.makedirs(config_dir)
+        except OSError as exc:
+            if exc.errno != errno.EEXIST:
+                raise
 
         config_file_name = 'master'
         with salt.utils.fopen(self.get_config_file_path(config_file_name), 'r') as fhr:
@@ -136,15 +133,24 @@ class CopyTest(ShellCase, ShellCaseCommonTestsMixin):
                     yaml.dump(config, default_flow_style=False)
                 )
 
-        ret = self.run_script(
-            self._call_binary_,
-            '--out pprint --config-dir {0} \'*\' foo {0}/foo'.format(
-                config_dir
-            ),
-            catch_stderr=True,
-            with_retcode=True
-        )
         try:
+            fd_, fn_ = tempfile.mkstemp()
+            os.close(fd_)
+
+            with salt.utils.fopen(fn_, 'w') as fp_:
+                fp_.write('Hello world!\n')
+
+            ret = self.run_script(
+                self._call_binary_,
+                '--out pprint --config-dir {0} \'*\' {1} {0}/{2}'.format(
+                    config_dir,
+                    fn_,
+                    os.path.basename(fn_),
+                ),
+                catch_stderr=True,
+                with_retcode=True
+            )
+
             self.assertIn('minion', '\n'.join(ret[0]))
             self.assertIn('sub_minion', '\n'.join(ret[0]))
             self.assertFalse(os.path.isdir(os.path.join(config_dir, 'file:')))
@@ -158,7 +164,10 @@ class CopyTest(ShellCase, ShellCaseCommonTestsMixin):
             )
             self.assertEqual(ret[2], 2)
         finally:
-            if old_cwd is not None:
-                self.chdir(old_cwd)
+            try:
+                os.remove(fn_)
+            except OSError as exc:
+                if exc.errno != errno.ENOENT:
+                    raise
             if os.path.isdir(config_dir):
                 shutil.rmtree(config_dir)

tests/integration/states/test_pkg.py

@@ -32,7 +32,7 @@ __testcontext__ = {}
 _PKG_TARGETS = {
     'Arch': ['sl', 'libpng'],
     'Debian': ['python-plist', 'apg'],
-    'RedHat': ['xz-devel', 'zsh-html'],
+    'RedHat': ['units', 'zsh-html'],
     'FreeBSD': ['aalib', 'pth'],
     'Suse': ['aalib', 'python-pssh'],
     'MacOS': ['libpng', 'jpeg'],