Merge branch '2015.8' into '2016.3'

Conflicts:
  - salt/modules/aptpkg.py
  - tests/integration/__init__.py

doc/ref/configuration/master.rst

@@ -1232,6 +1232,10 @@ Example:
       - roots
       - git
 
+.. note::
+    For masterless Salt, this parameter must be specified in the minion config
+    file.
+
 .. conf_master:: fileserver_followsymlinks
 
 ``fileserver_followsymlinks``
@@ -1399,6 +1403,10 @@ Example:
         - /srv/salt/prod/services
         - /srv/salt/prod/states
 
+.. note::
+    For masterless Salt, this parameter must be specified in the minion config
+    file.
+
 git: Git Remote File Server Backend
 -----------------------------------
 

doc/ref/file_server/all/salt.fileserver.azurefs.rst

@@ -3,4 +3,3 @@ salt.fileserver.azurefs
 =======================
 
 .. automodule:: salt.fileserver.azurefs
-    :members:

doc/ref/file_server/all/salt.fileserver.gitfs.rst

@@ -3,4 +3,3 @@ salt.fileserver.gitfs
 =====================
 
 .. automodule:: salt.fileserver.gitfs
-    :members:

doc/ref/file_server/all/salt.fileserver.hgfs.rst

@@ -3,4 +3,3 @@ salt.fileserver.hgfs
 ====================
 
 .. automodule:: salt.fileserver.hgfs
-    :members:

doc/ref/file_server/all/salt.fileserver.minionfs.rst

@@ -3,4 +3,3 @@ salt.fileserver.minionfs
 ========================
 
 .. automodule:: salt.fileserver.minionfs
-    :members:

doc/ref/file_server/all/salt.fileserver.roots.rst

@@ -3,4 +3,3 @@ salt.fileserver.roots
 =====================
 
 .. automodule:: salt.fileserver.roots
-    :members:

doc/ref/file_server/all/salt.fileserver.s3fs.rst

@@ -3,4 +3,3 @@ salt.fileserver.s3fs
 ====================
 
 .. automodule:: salt.fileserver.s3fs
-    :members:

doc/ref/file_server/all/salt.fileserver.svnfs.rst

@@ -3,4 +3,3 @@ salt.fileserver.svnfs
 =====================
 
 .. automodule:: salt.fileserver.svnfs
-    :members:

doc/ref/file_server/backends.rst

@@ -50,8 +50,8 @@ With this configuration, the environments and files defined in the
 not found then the git repositories defined in :conf_master:`gitfs_remotes`
 will be searched.
 
-Environments
-------------
+Defining Environments
+---------------------
 
 Just as the order of the values in :conf_master:`fileserver_backend` matters,
 so too does the order in which different sources are defined within a

doc/ref/file_server/environments.rst

@@ -0,0 +1,89 @@
+.. _file-server-environments:
+
+===========================================
+Requesting Files from Specific Environments
+===========================================
+
+The Salt fileserver supports multiple environments, allowing for SLS files and
+other files to be isolated for better organization.
+
+For the default backend (called :py:mod:`roots <salt.fileserver.roots>`),
+environments are defined using the :conf_master:`roots <file_roots>` option.
+Other backends (such as :py:mod:`gitfs <salt.fileserver.gitfs>`) define
+environments in their own ways. For a list of available fileserver backends,
+see :ref:`here <all-salt.fileserver>`.
+
+.. _querystring-syntax:
+
+Querystring Syntax
+==================
+
+Any ``salt://`` file URL can specify its fileserver environment using a
+querystring syntax, like so:
+
+.. code-block:: bash
+
+    salt://path/to/file?saltenv=foo
+
+In :ref:`Reactor <reactor>` configurations, this method must be used to pull
+files from an environment other than ``base``.
+
+In States
+=========
+
+Minions can be instructed which environment to use both globally, and for a
+single state, and multiple methods for each are available:
+
+Globally
+--------
+
+A minion can be pinned to an environment using the :conf_minion:`environment`
+option in the minion config file.
+
+Additionally, the environment can be set for a single call to the following
+functions:
+
+- :py:mod:`state.apply <salt.modules.state.apply>`
+- :py:mod:`state.highstate <salt.modules.state.highstate>`
+- :py:mod:`state.sls <salt.modules.state.sls>`
+- :py:mod:`state.top <salt.modules.state.top>`
+
+.. note::
+    When the ``saltenv`` parameter is used to trigger a :ref:`highstate
+    <running-highstate>` using either :py:mod:`state.apply
+    <salt.modules.state.apply>` or :py:mod:`state.highstate
+    <salt.modules.state.highstate>`, only states from that environment will be
+    applied.
+
+On a Per-State Basis
+--------------------
+
+Within an individual state, there are two ways of specifying the environment.
+The first is to add a ``saltenv`` argument to the state. This example will pull
+the file from the ``config`` environment:
+
+.. code-block:: yaml
+
+    /etc/foo/bar.conf:
+      file.managed:
+        - source: salt://foo/bar.conf
+        - user: foo
+        - mode: 600
+        - saltenv: config
+
+Another way of doing the same thing is to use the :ref:`querystring syntax
+<querystring-syntax>` described above:
+
+.. code-block:: yaml
+
+    /etc/foo/bar.conf:
+      file.managed:
+        - source: salt://foo/bar.conf?saltenv=config
+        - user: foo
+        - mode: 600
+
+.. note::
+    Specifying the environment using either of the above methods is only
+    necessary in cases where a state from one environment needs to access files
+    from another environment. If the SLS file containing this state was in the
+    ``config`` environment, then it would look in that environment by default.

doc/topics/reactor/index.rst

@@ -57,8 +57,13 @@ and each event tag has a list of reactor SLS files to be run.
         - /srv/reactor/destroy/*.sls    # Globs can be used to match file names
 
       - 'myco/custom/event/tag':        # React to custom event tags
-        - salt://reactor/mycustom.sls   # Put reactor files under file_roots
+        - salt://reactor/mycustom.sls   # Reactor files can come from the salt fileserver
 
+.. note::
+    In the above example, ``salt://reactor/mycustom.sls`` refers to the
+    ``base`` environment. To pull this file from a different environment, use
+    the :ref:`querystring syntax <querystring-syntax>` (e.g.
+    ``salt://reactor/mycustom.sls?saltenv=reactor``).
 
 Reactor sls files are similar to state and pillar sls files.  They are
 by default yaml + Jinja templates and are passed familiar context variables.

doc/topics/tutorials/states_pt4.rst

@@ -78,6 +78,12 @@ environments, allowing them to be pushed to QA hosts and tested.
 Finally, if moved to the same relative path within ``/srv/salt/prod``, the
 files are now available in all three environments.
 
+Requesting files from specific fileserver environments
+======================================================
+
+See :ref:`here <file-server-environments>` for documentation on how to request
+files from specific environments.
+
 Practical Example
 =================
 

salt/modules/aptpkg.py

@@ -1293,18 +1293,30 @@ def upgrade_available(name):
     return latest_version(name) != ''
 
 
-def version_cmp(pkg1, pkg2):
+def version_cmp(pkg1, pkg2, ignore_epoch=False):
     '''
     Do a cmp-style comparison on two packages. Return -1 if pkg1 < pkg2, 0 if
     pkg1 == pkg2, and 1 if pkg1 > pkg2. Return None if there was a problem
     making the comparison.
 
+    ignore_epoch : False
+        Set to ``True`` to ignore the epoch when comparing versions
+
+        .. versionadded:: 2015.8.10,2016.3.2
+
     CLI Example:
 
     .. code-block:: bash
 
         salt '*' pkg.version_cmp '0.2.4-0ubuntu1' '0.2.4.1-0ubuntu1'
     '''
+    normalize = lambda x: str(x).split(':', 1)[-1] if ignore_epoch else str(x)
+    # both apt_pkg.version_compare and _cmd_quote need string arguments.
+    pkg1 = normalize(pkg1)
+    pkg2 = normalize(pkg2)
+
+    # if we have apt_pkg, this will be quickier this way
+    # and also do not rely on shell.
     if HAS_APTPKG:
         try:
             # the apt_pkg module needs to be manually initialized

salt/modules/cp.py

@@ -195,6 +195,20 @@ def get_file(path,
     Use the *gzip* named argument to enable it.  Valid values are 1..9, where 1
     is the lightest compression and 9 the heaviest.  1 uses the least CPU on
     the master (and minion), 9 uses the most.
+
+    There are two ways of defining the fileserver environment (a.k.a.
+    ``saltenv``) from which to retrieve the file. One is to use the ``saltenv``
+    parameter, and the other is to use a querystring syntax in the ``salt://``
+    URL. The below two examples are equivalent:
+
+    .. code-block:: bash
+
+        salt '*' cp.get_file salt://foo/bar.conf /etc/foo/bar.conf saltenv=config
+        salt '*' cp.get_file salt://foo/bar.conf?saltenv=config /etc/foo/bar.conf
+
+    .. note::
+        It may be necessary to quote the URL when using the querystring method,
+        depending on the shell being used to run the command.
     '''
     if env is not None:
         salt.utils.warn_until(
@@ -357,6 +371,20 @@ def cache_file(path, saltenv='base', env=None):
     .. code-block:: bash
 
         salt '*' cp.cache_file salt://path/to/file
+
+    There are two ways of defining the fileserver environment (a.k.a.
+    ``saltenv``) from which to cache the file. One is to use the ``saltenv``
+    parameter, and the other is to use a querystring syntax in the ``salt://``
+    URL. The below two examples are equivalent:
+
+    .. code-block:: bash
+
+        salt '*' cp.cache_file salt://foo/bar.conf saltenv=config
+        salt '*' cp.cache_file salt://foo/bar.conf?saltenv=config
+
+    .. note::
+        It may be necessary to quote the URL when using the querystring method,
+        depending on the shell being used to run the command.
     '''
     if env is not None:
         salt.utils.warn_until(
@@ -415,6 +443,30 @@ def cache_files(paths, saltenv='base', env=None):
     .. code-block:: bash
 
         salt '*' cp.cache_files salt://pathto/file1,salt://pathto/file1
+
+    There are two ways of defining the fileserver environment (a.k.a.
+    ``saltenv``) from which to cache the files. One is to use the ``saltenv``
+    parameter, and the other is to use a querystring syntax in the ``salt://``
+    URL. The below two examples are equivalent:
+
+    .. code-block:: bash
+
+        salt '*' cp.cache_files salt://foo/bar.conf,salt://foo/baz.conf saltenv=config
+        salt '*' cp.cache_files salt://foo/bar.conf?saltenv=config,salt://foo/baz.conf?saltenv=config
+
+    The querystring method is less useful when all files are being cached from
+    the same environment, but is a good way of caching files from multiple
+    different environments in the same command. For example, the below command
+    will cache the first file from the ``config1`` environment, and the second
+    one from the ``config2`` environment.
+
+    .. code-block:: bash
+
+        salt '*' cp.cache_files salt://foo/bar.conf?saltenv=config1,salt://foo/bar.conf?saltenv=config2
+
+    .. note::
+        It may be necessary to quote the URL when using the querystring method,
+        depending on the shell being used to run the command.
     '''
     if env is not None:
         salt.utils.warn_until(

salt/modules/dockerng.py

@@ -105,17 +105,18 @@ Both methods can be combined; any registry configured under
 Configuration Options
 ---------------------
 
-The following options can be set in the :ref:`minion config
-<configuration-salt-minion>`:
+The following configuration options can be set to fine-tune how Salt uses
+Docker:
 
 - ``docker.url``: URL to the docker service (default: local socket).
-- ``docker.version``: API version to use (default: currently 1.4 API).
-- ``docker.exec_driver``: Execution driver to use, one of the following:
-    - nsenter
-    - lxc-attach
-    - docker-exec
+- ``docker.version``: API version to use
+- ``docker.exec_driver``: Execution driver to use, one of ``nsenter``,
+  ``lxc-attach``, or ``docker-exec``. See the :ref:`Executing Commands Within a
+  Running Container <docker-execution-driver>` section for more details on how
+  this config parameter is used.
 
-    See :ref:`Executing Commands Within a Running Container <docker-execution-driver>`.
+These configuration options are retrieved using :py:mod:`config.get
+<salt.modules.config.get>` (click the link for further information).
 
 Functions
 ---------
@@ -802,6 +803,14 @@ def _get_exec_driver():
             __context__[contextkey] = 'lxc-attach'
         elif driver.startswith('native-') and HAS_NSENTER:
             __context__[contextkey] = 'nsenter'
+        elif not driver.strip() and HAS_NSENTER:
+            log.warning(
+                'ExecutionDriver from \'docker info\' is blank, falling '
+                'back to using \'nsenter\'. To squelch this warning, set '
+                'docker.exec_driver. See the Salt documentation for the '
+                'dockerng module for more information.'
+            )
+            __context__[contextkey] = 'nsenter'
         else:
             raise NotImplementedError(
                 'Unknown docker ExecutionDriver \'{0}\', or didn\'t find '

salt/modules/ebuild.py

@@ -992,7 +992,7 @@ def depclean(name=None, slot=None, fromrepo=None, pkgs=None):
     return salt.utils.compare_dicts(old, new)
 
 
-def version_cmp(pkg1, pkg2):
+def version_cmp(pkg1, pkg2, **kwargs):
     '''
     Do a cmp-style comparison on two packages. Return -1 if pkg1 < pkg2, 0 if
     pkg1 == pkg2, and 1 if pkg1 > pkg2. Return None if there was a problem
@@ -1004,6 +1004,16 @@ def version_cmp(pkg1, pkg2):
 
         salt '*' pkg.version_cmp '0.2.4-0' '0.2.4.1-0'
     '''
+    # ignore_epoch is not supported here, but has to be included for API
+    # compatibility. Rather than putting this argument into the function
+    # definition (and thus have it show up in the docs), we just pop it out of
+    # the kwargs dict and then raise an exception if any kwargs other than
+    # ignore_epoch were passed.
+    kwargs = salt.utils.clean_kwargs(**kwargs)
+    kwargs.pop('ignore_epoch', None)
+    if kwargs:
+        salt.utils.invalid_kwargs(kwargs)
+
     regex = r'^~?([^:\[]+):?[^\[]*\[?.*$'
     ver1 = re.match(regex, pkg1)
     ver2 = re.match(regex, pkg2)

salt/modules/rpm.py

@@ -575,7 +575,7 @@ def info(*packages, **attr):
     return ret
 
 
-def version_cmp(ver1, ver2):
+def version_cmp(ver1, ver2, ignore_epoch=False):
     '''
     .. versionadded:: 2015.8.9
 
@@ -583,12 +583,21 @@ def version_cmp(ver1, ver2):
     ver1 == ver2, and 1 if ver1 > ver2. Return None if there was a problem
     making the comparison.
 
+    ignore_epoch : False
+        Set to ``True`` to ignore the epoch when comparing versions
+
+        .. versionadded:: 2015.8.10,2016.3.2
+
     CLI Example:
 
     .. code-block:: bash
 
         salt '*' pkg.version_cmp '0.2-001' '0.2.0.1-002'
     '''
+    normalize = lambda x: str(x).split(':', 1)[-1] if ignore_epoch else str(x)
+    ver1 = normalize(ver1)
+    ver2 = normalize(ver2)
+
     try:
         cmp_func = None
         if HAS_RPM:
@@ -661,7 +670,10 @@ def version_cmp(ver1, ver2):
             ver1, ver2, exc
         )
 
-    return salt.utils.version_cmp(ver1, ver2)
+    # We would already have normalized the versions at the beginning of this
+    # function if ignore_epoch=True, so avoid unnecessary work and just pass
+    # False for this value.
+    return salt.utils.version_cmp(ver1, ver2, ignore_epoch=False)
 
 
 def checksum(*paths):

salt/modules/yumpkg.py

@@ -519,7 +519,7 @@ def version(*names, **kwargs):
     return __salt__['pkg_resource.version'](*names, **kwargs)
 
 
-def version_cmp(pkg1, pkg2):
+def version_cmp(pkg1, pkg2, ignore_epoch=False):
     '''
     .. versionadded:: 2015.5.4
 
@@ -527,6 +527,11 @@ def version_cmp(pkg1, pkg2):
     pkg1 == pkg2, and 1 if pkg1 > pkg2. Return None if there was a problem
     making the comparison.
 
+    ignore_epoch : False
+        Set to ``True`` to ignore the epoch when comparing versions
+
+        .. versionadded:: 2015.8.10,2016.3.2
+
     CLI Example:
 
     .. code-block:: bash
@@ -534,7 +539,7 @@ def version_cmp(pkg1, pkg2):
         salt '*' pkg.version_cmp '0.2-001' '0.2.0.1-002'
     '''
 
-    return __salt__['lowpkg.version_cmp'](pkg1, pkg2)
+    return __salt__['lowpkg.version_cmp'](pkg1, pkg2, ignore_epoch=ignore_epoch)
 
 
 def list_pkgs(versions_as_list=False, **kwargs):

salt/modules/zypper.py

@@ -526,7 +526,7 @@ def version(*names, **kwargs):
     return __salt__['pkg_resource.version'](*names, **kwargs) or {}
 
 
-def version_cmp(ver1, ver2):
+def version_cmp(ver1, ver2, ignore_epoch=False):
     '''
     .. versionadded:: 2015.5.4
 
@@ -534,13 +534,18 @@ def version_cmp(ver1, ver2):
     ver1 == ver2, and 1 if ver1 > ver2. Return None if there was a problem
     making the comparison.
 
+    ignore_epoch : False
+        Set to ``True`` to ignore the epoch when comparing versions
+
+        .. versionadded:: 2015.8.10,2016.3.2
+
     CLI Example:
 
     .. code-block:: bash
 
         salt '*' pkg.version_cmp '0.2-001' '0.2.0.1-002'
     '''
-    return __salt__['lowpkg.version_cmp'](str(ver1), str(ver2))
+    return __salt__['lowpkg.version_cmp'](ver1, ver2, ignore_epoch=ignore_epoch)
 
 
 def list_pkgs(versions_as_list=False, **kwargs):

salt/states/dockerng.py

@@ -444,6 +444,8 @@ def _compare(actual, create_kwargs, defaults_from_image):
             # "actual" dict sorted(somedict) still just gives you a sorted
             # list of the dictionary's keys. And we don't care about the
             # value for "volumes", just its keys.
+            if actual_data is None:
+                actual_data = []
             actual_data = sorted(actual_data)
             desired_data = sorted(data)
             log.trace('dockerng.running ({0}): munged actual value: {1}'

salt/states/pkg.py

@@ -151,13 +151,13 @@ def _fulfills_version_spec(versions, oper, desired_version,
     Returns True if any of the installed versions match the specified version,
     otherwise returns False
     '''
-    normalize = lambda x: x.split(':', 1)[-1] if ignore_epoch else x
     cmp_func = __salt__.get('pkg.version_cmp')
     for ver in versions:
-        if salt.utils.compare_versions(ver1=normalize(ver),
+        if salt.utils.compare_versions(ver1=ver,
                                        oper=oper,
-                                       ver2=normalize(desired_version),
-                                       cmp_func=cmp_func):
+                                       ver2=desired_version,
+                                       cmp_func=cmp_func,
+                                       ignore_epoch=ignore_epoch):
             return True
     return False
 

salt/utils/__init__.py

@@ -2287,7 +2287,7 @@ def kwargs_warn_until(kwargs,
         )
 
 
-def version_cmp(pkg1, pkg2):
+def version_cmp(pkg1, pkg2, ignore_epoch=False):
     '''
     Compares two version strings using distutils.version.LooseVersion. This is
     a fallback for providers which don't have a version comparison utility
@@ -2295,6 +2295,10 @@ def version_cmp(pkg1, pkg2):
     version2, and 1 if version1 > version2. Return None if there was a problem
     making the comparison.
     '''
+    normalize = lambda x: str(x).split(':', 1)[-1] if ignore_epoch else str(x)
+    pkg1 = normalize(pkg1)
+    pkg2 = normalize(pkg2)
+
     try:
         # pylint: disable=no-member
         if distutils.version.LooseVersion(pkg1) < \
@@ -2311,22 +2315,25 @@ def version_cmp(pkg1, pkg2):
     return None
 
 
-def compare_versions(ver1='', oper='==', ver2='', cmp_func=None):
+def compare_versions(ver1='',
+                     oper='==',
+                     ver2='',
+                     cmp_func=None,
+                     ignore_epoch=False):
     '''
     Compares two version numbers. Accepts a custom function to perform the
     cmp-style version comparison, otherwise uses version_cmp().
     '''
     cmp_map = {'<': (-1,), '<=': (-1, 0), '==': (0,),
                '>=': (0, 1), '>': (1,)}
-    if oper not in ['!='] and oper not in cmp_map:
-        log.error('Invalid operator "{0}" for version '
-                  'comparison'.format(oper))
+    if oper not in ('!=',) and oper not in cmp_map:
+        log.error('Invalid operator \'%s\' for version comparison', oper)
         return False
 
     if cmp_func is None:
         cmp_func = version_cmp
 
-    cmp_result = cmp_func(ver1, ver2)
+    cmp_result = cmp_func(ver1, ver2, ignore_epoch=ignore_epoch)
     if cmp_result is None:
         return False
 

tests/integration/states/ssh.py

@@ -24,7 +24,7 @@ import salt.utils
 
 KNOWN_HOSTS = os.path.join(integration.TMP, 'known_hosts')
 GITHUB_FINGERPRINT = '16:27:ac:a5:76:28:2d:36:63:1b:56:4d:eb:df:a6:48'
-GITHUB_IP = '192.30.252.129'
+GITHUB_IP = '192.30.253.113'
 
 
 @skip_if_binaries_missing(['ssh', 'ssh-keygen'], check_all=True)

tests/unit/modules/dockerng_test.py

@@ -63,6 +63,7 @@ class DockerngTestCase(TestCase):
                 all=True,
                 filters={'label': 'KEY'})
 
+    @skipIf(_docker_py_version() is None, 'docker-py needs to be installed for this test to run')
     @patch.object(dockerng_mod, '_get_exec_driver')
     def test_check_mine_cache_is_refreshed_on_container_change_event(self, _):
         '''