Merge branch 'develop' of https://github.com/saltstack/salt into develop

.mention-bot

@@ -1,7 +1,5 @@
 {
  "skipTitle": "Merge forward",
- "delayed": true,
- "delayedUntil": "2h",
  "userBlacklist": []
 }
 

.pylintrc

@@ -25,7 +25,7 @@ load-plugins=saltpylint.pep8,
   saltpylint.py3modernize,
   saltpylint.smartup,
   saltpylint.minpyver,
-  saltpylint.salttesting,
+  saltpylint.blacklist,
   saltpylint.thirdparty
 
 # Use multiple processes to speed up Pylint.
@@ -43,7 +43,7 @@ extension-pkg-whitelist=
 
 # Fileperms Lint Plugin Settings
 fileperms-default=0644
-fileperms-ignore-paths=tests/runtests.py,tests/jenkins*.py,tests/saltsh.py,tests/buildpackage.py
+fileperms-ignore-paths=setup.py,tests/runtests.py,tests/jenkins*.py,tests/saltsh.py,tests/buildpackage.py
 
 # Minimum Python Version To Enforce
 minimum-python-version = 2.7

.testing.pylintrc

@@ -22,7 +22,7 @@ load-plugins=saltpylint.pep8,
   saltpylint.py3modernize,
   saltpylint.smartup,
   saltpylint.minpyver,
-  saltpylint.salttesting,
+  saltpylint.blacklist,
   saltpylint.thirdparty
 
 # Use multiple processes to speed up Pylint.
@@ -40,7 +40,7 @@ extension-pkg-whitelist=
 
 # Fileperms Lint Plugin Settings
 fileperms-default=0644
-fileperms-ignore-paths=tests/runtests.py,tests/jenkins*.py,tests/saltsh.py,tests/buildpackage.py
+fileperms-ignore-paths=setup.py,tests/runtests.py,tests/jenkins*.py,tests/saltsh.py,tests/buildpackage.py
 
 # Minimum Python Version To Enforce
 minimum-python-version = 2.7

doc/ref/modules/all/index.rst

@@ -70,6 +70,7 @@ execution modules
     bsd_shadow
     btrfs
     cabal
+    capirca_acl
     cassandra
     cassandra_cql
     celery
@@ -248,6 +249,7 @@ execution modules
     namecheap_ssl
     namecheap_users
     napalm
+    napalm_acl
     napalm_bgp
     napalm_network
     napalm_ntp

doc/ref/modules/all/salt.modules.capirca_acl.rst

@@ -0,0 +1,5 @@
+salt.modules.capirca_acl module
+===============================
+
+.. automodule:: salt.modules.capirca_acl
+    :members:

doc/ref/modules/all/salt.modules.napalm_acl.rst

@@ -0,0 +1,7 @@
+==============================
+salt.modules.napalm_acl module
+==============================
+
+.. automodule:: salt.modules.napalm_acl
+    :members:
+

doc/ref/returners/all/index.rst

@@ -22,6 +22,7 @@ returner modules
     hipchat_return
     influxdb_return
     kafka_return
+    librato_return
     local
     local_cache
     mattermost_returner

doc/ref/returners/all/salt.returners.librato_return.rst

@@ -0,0 +1,6 @@
+=============================
+salt.returners.librato_return
+=============================
+
+.. automodule:: salt.returners.librato_return
+    :members:

doc/ref/states/all/index.rst

@@ -158,6 +158,7 @@ state modules
     mysql_grants
     mysql_query
     mysql_user
+    netacl
     netconfig
     netntp
     netsnmp

doc/ref/states/all/salt.states.netacl.rst

@@ -0,0 +1,7 @@
+==================
+salt.states.netacl
+==================
+
+.. automodule:: salt.states.netacl
+    :members:
+

requirements/dev_python27.txt

@@ -8,4 +8,5 @@ moto>=0.3.6
 SaltPyLint>=v2017.3.6
 GitPython>=0.3
 pytest
+git+https://github.com/eisensheng/pytest-catchlog.git@develop#egg=Pytest-catchlog
 git+https://github.com/saltstack/pytest-salt.git@master#egg=pytest-salt

requirements/pytest.txt

@@ -1,4 +1,3 @@
 pytest
-git+https://github.com/eisensheng/pytest-catchlog.git@develop#egg=Pytest-catchlog
 pytest-helpers-namespace
 pytest-tempdir

salt/beacons/ps.py

@@ -60,18 +60,19 @@ def beacon(config):
         if _name not in procs:
             procs.append(_name)
 
-    for process in config:
-        ret_dict = {}
-        if config[process] == 'running':
-            if process not in procs:
-                ret_dict[process] = 'Stopped'
-                ret.append(ret_dict)
-        elif config[process] == 'stopped':
-            if process in procs:
-                ret_dict[process] = 'Running'
-                ret.append(ret_dict)
-        else:
-            if process not in procs:
-                ret_dict[process] = False
-                ret.append(ret_dict)
+    for entry in config:
+        for process in entry:
+            ret_dict = {}
+            if entry[process] == 'running':
+                if process not in procs:
+                    ret_dict[process] = 'Stopped'
+                    ret.append(ret_dict)
+            elif entry[process] == 'stopped':
+                if process in procs:
+                    ret_dict[process] = 'Running'
+                    ret.append(ret_dict)
+            else:
+                if process not in procs:
+                    ret_dict[process] = False
+                    ret.append(ret_dict)
     return ret

salt/beacons/status.py

@@ -126,33 +126,42 @@ def beacon(config):
         }]
 
     if len(config) < 1:
-        config = {
+        config = [{
             'loadavg': ['all'],
             'cpustats': ['all'],
             'meminfo': ['all'],
             'vmstats': ['all'],
             'time': ['all'],
-        }
+        }]
 
-    for func in config:
-        try:
-            data = __salt__['status.{0}'.format(func)]()
-        except salt.exceptions.CommandExecutionError as exc:
-            log.debug('Status beacon attempted to process function {0} \
-                    but encountered error: {1}'.format(func, exc))
-            continue
-        ret[func] = {}
-        item = config[func]
-        if item == 'all':
-            ret[func] = data
-        else:
+    if not isinstance(config, list):
+        # To support the old dictionary config format
+        config = [config]
+
+    for entry in config:
+        for func in entry:
+            ret[func] = {}
             try:
-                try:
-                    ret[func][item] = data[item]
-                except TypeError:
-                    ret[func][item] = data[int(item)]
-            except KeyError as exc:
-                ret[func] = 'Status beacon is incorrectly configured: {0}'.format(exc)
+                data = __salt__['status.{0}'.format(func)]()
+            except salt.exceptions.CommandExecutionError as exc:
+                log.debug('Status beacon attempted to process function {0} '
+                          'but encountered error: {1}'.format(func, exc))
+                continue
+            if not isinstance(entry[func], list):
+                func_items = [entry[func]]
+            else:
+                func_items = entry[func]
+            for item in func_items:
+                if item == 'all':
+                    ret[func] = data
+                else:
+                    try:
+                        try:
+                            ret[func][item] = data[item]
+                        except TypeError:
+                            ret[func][item] = data[int(item)]
+                    except KeyError as exc:
+                        ret[func] = 'Status beacon is incorrectly configured: {0}'.format(exc)
 
     return [{
         'tag': ctime,

salt/client/ssh/client.py

@@ -5,6 +5,7 @@ from __future__ import absolute_import
 import os
 import copy
 import logging
+import random
 
 # Import Salt libs
 import salt.config
@@ -196,3 +197,53 @@ class SSHClient(object):
         '''
         # TODO Not implemented
         raise SaltClientError
+
+    def cmd_subset(
+            self,
+            tgt,
+            fun,
+            arg=(),
+            timeout=None,
+            tgt_type='glob',
+            ret='',
+            kwarg=None,
+            sub=3,
+            **kwargs):
+        '''
+        Execute a command on a random subset of the targeted systems
+
+        The function signature is the same as :py:meth:`cmd` with the
+        following exceptions.
+
+        :param sub: The number of systems to execute on
+
+        .. code-block:: python
+
+            >>> import salt.client.ssh.client
+            >>> sshclient= salt.client.ssh.client.SSHClient()
+            >>> sshclient.cmd_subset('*', 'test.ping', sub=1)
+            {'jerry': True}
+
+        .. versionadded:: Nitrogen
+        '''
+        if 'expr_form' in kwargs:
+            salt.utils.warn_until(
+                'Fluorine',
+                'The target type should be passed using the \'tgt_type\' '
+                'argument instead of \'expr_form\'. Support for using '
+                '\'expr_form\' will be removed in Salt Fluorine.'
+            )
+            tgt_type = kwargs.pop('expr_form')
+        minion_ret = self.cmd(tgt,
+                              'sys.list_functions',
+                              tgt_type=tgt_type,
+                              **kwargs)
+        minions = list(minion_ret)
+        random.shuffle(minions)
+        f_tgt = []
+        for minion in minions:
+            if fun in minion_ret[minion]['return']:
+                f_tgt.append(minion)
+            if len(f_tgt) >= sub:
+                break
+        return self.cmd_iter(f_tgt, fun, arg, timeout, tgt_type='list', ret=ret, kwarg=kwarg, **kwargs)

salt/modules/boto_lambda.py

@@ -597,6 +597,25 @@ def get_permissions(FunctionName, Qualifier=None,
         return {'permissions': None, 'error': err}
 
 
+def list_functions(region=None, key=None, keyid=None, profile=None):
+    '''
+    List all Lambda functions visible in the current scope.
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt myminion boto_lambda.list_functions
+
+    '''
+    conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
+
+    ret = []
+    for funcs in salt.utils.boto3.paged_call(conn.list_functions):
+        ret += funcs['Functions']
+    return ret
+
+
 def list_function_versions(FunctionName,
                            region=None, key=None, keyid=None, profile=None):
     '''

salt/modules/napalm_acl.py

@@ -0,0 +1,814 @@
+# -*- coding: utf-8 -*-
+'''
+NAPALM ACL
+==========
+
+Generate and load ACL (firewall) configuration on network devices.
+
+.. versionadded:: Nitrogen
+
+:codeauthor: Mircea Ulinic <mircea@cloudflare.com>
+:maturity:   new
+:depends:    capirca, napalm
+:platform:   unix
+
+Dependencies
+------------
+
+The firewall configuration is generated by Capirca_.
+
+.. _Capirca: https://github.com/google/capirca
+
+To be able to load configuration on network devices,
+it requires NAPALM_ library to be installed:  ``pip install napalm``.
+Please check Installation_ for complete details.
+
+.. _NAPALM: https://napalm.readthedocs.io
+.. _Installation: https://napalm.readthedocs.io/en/latest/installation.html
+'''
+from __future__ import absolute_import
+
+import logging
+log = logging.getLogger(__file__)
+
+# Import third party libs
+try:
+    # pylint: disable=W0611
+    import aclgen
+    HAS_CAPIRCA = True
+    # pylint: enable=W0611
+except ImportError:
+    HAS_CAPIRCA = False
+
+try:
+    # pylint: disable=W0611
+    import napalm_base
+    # pylint: enable=W0611
+    HAS_NAPALM = True
+except ImportError:
+    HAS_NAPALM = False
+
+# import Salt modules
+from salt.utils.napalm import proxy_napalm_wrap
+
+# ------------------------------------------------------------------------------
+# module properties
+# ------------------------------------------------------------------------------
+
+__virtualname__ = 'netacl'
+__proxyenabled__ = ['napalm']
+# allow napalm proxy only
+
+# ------------------------------------------------------------------------------
+# property functions
+# ------------------------------------------------------------------------------
+
+
+def __virtual__():
+    '''
+    This module requires both NAPALM and Capirca.
+    '''
+    if HAS_CAPIRCA and HAS_NAPALM:
+        return __virtualname__
+    else:
+        return (False, 'The netacl (napalm_acl) module cannot be loaded: \
+                Please install capirca and napalm.')
+
+# ------------------------------------------------------------------------------
+# helper functions -- will not be exported
+# ------------------------------------------------------------------------------
+
+
+def _get_capirca_platform():  # pylint: disable=too-many-return-statements
+    '''
+    Given the following NAPALM grains, we can determine the Capirca platform name:
+
+    - vendor
+    - device model
+    - operating system
+
+    Not the most optimal.
+    '''
+    vendor = __grains__['vendor'].lower()
+    os_ = __grains__['os'].lower()
+    model = __grains__['model'].lower()
+    if vendor == 'juniper' and 'srx' in model:
+        return 'junipersrx'
+    elif vendor == 'cisco' and os_ == 'ios':
+        return 'cisco'
+    elif vendor == 'cisco' and os_ == 'iosxr':
+        return 'ciscoxr'
+    elif vendor == 'cisco' and os_ == 'asa':
+        return 'ciscoasa'
+    elif os_ == 'linux':
+        return 'iptables'
+    elif vendor == 'palo alto networks':
+        return 'paloaltofw'
+    # anything else will point to the vendor
+    # i.e.: some of the Capirca platforms are named by the device vendor
+    # e.g.: eOS => arista, junos => juniper, etc.
+    return vendor
+
+# ------------------------------------------------------------------------------
+# callable functions
+# ------------------------------------------------------------------------------
+
+
+@proxy_napalm_wrap
+def load_term_config(filter_name,
+                     term_name,
+                     filter_options=None,
+                     pillar_key='acl',
+                     pillarenv=None,
+                     saltenv=None,
+                     merge_pillar=True,
+                     revision_id=None,
+                     revision_no=None,
+                     revision_date=True,
+                     revision_date_format='%Y/%m/%d',
+                     test=False,
+                     commit=True,
+                     debug=False,
+                     source_service=None,
+                     destination_service=None,
+                     **term_fields):
+    '''
+    Generate and load the configuration of a policy term.
+
+    filter_name
+        The name of the policy filter.
+
+    term_name
+        The name of the term.
+
+    filter_options
+        Additional filter options. These options are platform-specific.
+        See the complete list of options_.
+
+        .. _options: https://github.com/google/capirca/wiki/Policy-format#header-section
+
+    pillar_key: ``acl``
+        The key in the pillar containing the default attributes values. Default: ``acl``.
+        If the pillar contains the following structure:
+
+        .. code-block:: yaml
+
+            firewall:
+                my-filter:
+                    my-term:
+                        source_port: 1234
+                        source_address:
+                            - 1.2.3.4/32
+                            - 5.6.7.8/32
+
+        The ``pillar_key`` field would be specified as ``firewall``.
+
+    pillarenv
+        Query the master to generate fresh pillar data on the fly,
+        specifically from the requested pillar environment.
+
+    saltenv
+        Included only for compatibility with
+        :conf_minion:`pillarenv_from_saltenv`, and is otherwise ignored.
+
+    merge_pillar: ``True``
+        Merge the CLI variables with the pillar. Default: ``True``.
+
+    revision_id
+        Add a comment in the term config having the description for the changes applied.
+
+    revision_no
+        The revision count.
+
+    revision_date: ``True``
+        Boolean flag: display the date when the term configuration was generated. Default: ``True``.
+
+    revision_date_format: ``%Y/%m/%d``
+        The date format to be used when generating the perforce data. Default: ``%Y/%m/%d`` (<year>/<month>/<day>).
+
+    test: ``False``
+        Dry run? If set as ``True``, will apply the config, discard and return the changes.
+        Default: ``False`` and will commit the changes on the device.
+
+    commit: ``True``
+        Commit? Default: ``True``.
+
+    debug: ``False``
+        Debug mode. Will insert a new key under the output dictionary,
+        as ``loaded_config`` contaning the raw configuration loaded on the device.
+
+    source_service
+        A special service to choose from. This is a helper so the user is able to
+        select a source just using the name, instead of specifying a source_port and protocol.
+
+        As this module is available on Unix platforms only,
+        it reads the IANA_ port assignment from /etc/services.
+
+        If the user requires additional shortcuts to be referenced, they can add entries under /etc/services,
+        which can be managed using the :mod:`file state <salt.states.file>`.
+
+        .. _IANA: http://www.iana.org/assignments/port-numbers
+
+    destination_service
+        A special service to choose from. This is a helper so the user is able to
+        select a source just using the name, instead of specifying a destination_port and protocol.
+        Allows the same options as ``source_service``.
+
+    **term_fields
+        Term attributes.
+        To see what fields are supported, please consult the list of supported keywords_.
+            Some platforms have few other optional_ keyworkds.
+
+            .. _keywords:https://github.com/google/capirca/wiki/Policy-format#keywords
+            .. _optional: https://github.com/google/capirca/wiki/Policy-format#optionally-supported-keywords
+
+    .. note::
+        The following fields are accepted:
+
+        - action
+        - address
+        - address_exclude
+        - comment
+        - counter
+        - expiration
+        - destination_address
+        - destination_address_exclude
+        - destination_port
+        - destination_prefix
+        - forwarding_class
+        - forwarding_class_except
+        - logging
+        - log_name
+        - loss_priority
+        - option
+        - policer
+        - port
+        - precedence
+        - principals
+        - protocol
+        - protocol_except
+        - qos
+        - pan_application
+        - routing_instance
+        - source_address
+        - source_address_exclude
+        - source_port
+        - source_prefix
+        - verbatim
+        - packet_length
+        - fragment_offset
+        - hop_limit
+        - icmp_type
+        - ether_type
+        - traffic_class_count
+        - traffic_type
+        - translated
+        - dscp_set
+        - dscp_match
+        - dscp_except
+        - next_ip
+        - flexible_match_range
+        - source_prefix_except
+        - destination_prefix_except
+        - vpn
+        - source_tag
+        - destination_tag
+        - source_interface
+        - destination_interface
+        - flattened
+        - flattened_addr
+        - flattened_saddr
+        - flattened_daddr
+
+    .. note::
+        The following fields can be also a single value and a list of values:
+
+        - action
+        - address
+        - address_exclude
+        - comment
+        - destination_address
+        - destination_address_exclude
+        - destination_port
+        - destination_prefix
+        - forwarding_class
+        - forwarding_class_except
+        - logging
+        - option
+        - port
+        - precedence
+        - principals
+        - protocol
+        - protocol_except
+        - pan_application
+        - source_address
+        - source_address_exclude
+        - source_port
+        - source_prefix
+        - verbatim
+        - icmp_type
+        - ether_type
+        - traffic_type
+        - dscp_match
+        - dscp_except
+        - flexible_match_range
+        - source_prefix_except
+        - destination_prefix_except
+        - source_tag
+        - destination_tag
+        - source_service
+        - destination_service
+
+        Example: ``destination_address`` can be either defined as:
+
+        .. code-block:: yaml
+
+            destination_address: 172.17.17.1/24
+
+        or as a list of destination IP addresses:
+
+        .. code-block:: yaml
+
+            destination_address:
+                - 172.17.17.1/24
+                - 172.17.19.1/24
+
+        or a list of services to be matched:
+
+        .. code-block:: yaml
+
+            source_service:
+                - ntp
+                - snmp
+                - ldap
+                - bgpd
+
+    .. note::
+        The port fields ``source_port`` and ``destination_port`` can be used as above to select either
+        a single value, either a list of values, but also they can select port ranges. Example:
+
+        .. code-block:: yaml
+
+            source_port:
+                - [1000, 2000]
+                - [3000, 4000]
+
+        With the configuration above, the user is able to select the 1000-2000 and 3000-4000 source port ranges.
+
+    The output is a dictionary having the same form as :mod:`net.load_config <salt.modules.napalm_network.load_config>`.
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt 'edge01.bjm01' netacl.load_term_config filter-name term-name source_address=1.2.3.4 destination_address=5.6.7.8 action=accept test=True debug=True
+
+    Output Example:
+
+    .. code-block:: yaml
+
+        edge01.bjm01:
+            ----------
+            already_configured:
+                False
+            comment:
+                Configuration discarded.
+            diff:
+                [edit firewall]
+                +    family inet {
+                +        /*
+                +         ** $Id:$
+                +         ** $Date:$
+                +         ** $Revision:$
+                +         **
+                +         */
+                +        filter filter-name {
+                +            interface-specific;
+                +            term term-name {
+                +                from {
+                +                    source-address {
+                +                        1.2.3.4/32;
+                +                    }
+                +                    destination-address {
+                +                        5.6.7.8/32;
+                +                    }
+                +                }
+                +                then accept;
+                +            }
+                +        }
+                +    }
+            loaded_config:
+                firewall {
+                    family inet {
+                        replace:
+                        /*
+                        ** $Id:$
+                        ** $Date:$
+                        ** $Revision:$
+                        **
+                        */
+                        filter filter-name {
+                            interface-specific;
+                            term term-name {
+                                from {
+                                    source-address {
+                                        1.2.3.4/32;
+                                    }
+                                    destination-address {
+                                        5.6.7.8/32;
+                                    }
+                                }
+                                then accept;
+                            }
+                        }
+                    }
+                }
+            result:
+                True
+    '''
+    if not filter_options:
+        filter_options = []
+    platform = _get_capirca_platform()
+    term_config = __salt__['capirca.get_term_config'](platform,
+                                                      filter_name,
+                                                      term_name,
+                                                      filter_options=filter_options,
+                                                      pillar_key=pillar_key,
+                                                      pillarenv=pillarenv,
+                                                      saltenv=saltenv,
+                                                      merge_pillar=merge_pillar,
+                                                      revision_id=revision_id,
+                                                      revision_no=revision_no,
+                                                      revision_date=revision_date,
+                                                      revision_date_format=revision_date_format,
+                                                      source_service=source_service,
+                                                      destination_service=destination_service,
+                                                      **term_fields)
+    return __salt__['net.load_config'](text=term_config,
+                                       test=test,
+                                       commit=commit,
+                                       debug=debug,
+                                       inherit_napalm_device=napalm_device)  # pylint: disable=undefined-variable
+
+
+@proxy_napalm_wrap
+def load_filter_config(filter_name,
+                       filter_options=None,
+                       terms=None,
+                       pillar_key='acl',
+                       pillarenv=None,
+                       saltenv=None,
+                       merge_pillar=True,
+                       only_lower_merge=False,
+                       revision_id=None,
+                       revision_no=None,
+                       revision_date=True,
+                       revision_date_format='%Y/%m/%d',
+                       test=False,
+                       commit=True,
+                       debug=False):
+    '''
+    Generate and load the configuration of a policy filter.
+
+    filter_name
+        The name of the policy filter.
+
+    filter_options
+        Additional filter options. These options are platform-specific.
+        See the complete list of options_.
+
+        .. _options: https://github.com/google/capirca/wiki/Policy-format#header-section
+
+    terms
+        Dictionary of terms for this policy filter.
+        If not specified or empty, will try to load the configuration from the pillar,
+        unless ``merge_pillar`` is set as ``False``.
+
+    pillar_key: ``acl``
+        The key in the pillar containing the default attributes values. Default: ``acl``.
+
+    pillarenv
+        Query the master to generate fresh pillar data on the fly,
+        specifically from the requested pillar environment.
+
+    saltenv
+        Included only for compatibility with
+        :conf_minion:`pillarenv_from_saltenv`, and is otherwise ignored.
+
+    merge_pillar: ``True``
+        Merge the CLI variables with the pillar. Default: ``True``
+
+    only_lower_merge: ``False``
+        Specify if it should merge only the terms fields. Otherwise it will try
+        to merge also filters fields. Default: ``False``.
+        This option requires ``merge_pillar``, otherwise it is ignored.
+
+    revision_id
+        Add a comment in the filter config having the description for the changes applied.
+
+    revision_no
+        The revision count.
+
+    revision_date: ``True``
+        Boolean flag: display the date when the filter configuration was generated. Default: ``True``.
+
+    revision_date_format: ``%Y/%m/%d``
+        The date format to be used when generating the perforce data. Default: ``%Y/%m/%d`` (<year>/<month>/<day>).
+
+    test: ``False``
+        Dry run? If set as ``True``, will apply the config, discard and return the changes.
+        Default: ``False`` and will commit the changes on the device.
+
+    commit: ``True``
+        Commit? Default: ``True``.
+
+    debug: ``False``
+        Debug mode. Will insert a new key under the output dictionary,
+        as ``loaded_config`` contaning the raw configuration loaded on the device.
+
+    The output is a dictionary having the same form as :mod:`net.load_config <salt.modules.napalm_network.load_config>`.
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt 'edge01.bjm01' netacl.load_filter_config my-filter pillar_key=netacl debug=True
+
+    Output Example:
+
+    .. code-block:: yaml
+
+        edge01.bjm01:
+            ----------
+            already_configured:
+                False
+            comment:
+            diff:
+                [edit firewall]
+                +    family inet {
+                +        /*
+                +         ** $Id:$
+                +         ** $Date:$
+                +         ** $Revision:$
+                +         **
+                +         */
+                +        filter my-filter {
+                +            interface-specific;
+                +            term my-term {
+                +                from {
+                +                    source-port [ 1234 1235 ];
+                +                }
+                +                then {
+                +                    reject;
+                +                }
+                +            }
+                +            term my-other-term {
+                +                from {
+                +                    protocol tcp;
+                +                    source-port 5678-5680;
+                +                }
+                +                then accept;
+                +            }
+                +        }
+                +    }
+            loaded_config:
+                firewall {
+                    family inet {
+                        replace:
+                        /*
+                        ** $Id:$
+                        ** $Date:$
+                        ** $Revision:$
+                        **
+                        */
+                        filter my-filter {
+                            interface-specific;
+                            term my-term {
+                                from {
+                                    source-port [ 1234 1235 ];
+                                }
+                                then {
+                                    reject;
+                                }
+                            }
+                            term my-other-term {
+                                from {
+                                    protocol tcp;
+                                    source-port 5678-5680;
+                                }
+                                then accept;
+                            }
+                        }
+                    }
+                }
+            result:
+                True
+
+    The filter configuration has been loaded from the pillar, having the following structure:
+
+    .. code-block:: yaml
+
+        netacl:
+          my-filter:
+            my-term:
+              source_port: [1234, 1235]
+              action: reject
+            my-other-term:
+              source_port:
+                - [5678, 5680]
+              protocol: tcp
+              action: accept
+    '''
+    if not filter_options:
+        filter_options = []
+    if not terms:
+        terms = {}
+    platform = _get_capirca_platform()
+    filter_config = __salt__['capirca.get_filter_config'](platform,
+                                                          filter_name,
+                                                          terms=terms,
+                                                          filter_options=filter_options,
+                                                          pillar_key=pillar_key,
+                                                          pillarenv=pillarenv,
+                                                          saltenv=saltenv,
+                                                          merge_pillar=merge_pillar,
+                                                          only_lower_merge=only_lower_merge,
+                                                          revision_id=revision_id,
+                                                          revision_no=revision_no,
+                                                          revision_date=revision_date,
+                                                          revision_date_format=revision_date_format)
+    return __salt__['net.load_config'](text=filter_config,
+                                       test=test,
+                                       commit=commit,
+                                       debug=debug,
+                                       inherit_napalm_device=napalm_device)  # pylint: disable=undefined-variable
+
+
+@proxy_napalm_wrap
+def load_policy_config(filters=None,
+                       pillar_key='acl',
+                       pillarenv=None,
+                       saltenv=None,
+                       merge_pillar=True,
+                       only_lower_merge=False,
+                       revision_id=None,
+                       revision_no=None,
+                       revision_date=True,
+                       revision_date_format='%Y/%m/%d',
+                       test=False,
+                       commit=True,
+                       debug=False):
+    '''
+    Generate and load the configuration of the whole policy.
+
+    filters
+        Dictionary of filters for this policy.
+        If not specified or empty, will try to load the configuration from the pillar,
+        unless ``merge_pillar`` is set as ``False``.
+
+    pillar_key: ``acl``
+        The key in the pillar containing the default attributes values. Default: ``acl``.
+
+    pillarenv
+        Query the master to generate fresh pillar data on the fly,
+        specifically from the requested pillar environment.
+
+    saltenv
+        Included only for compatibility with
+        :conf_minion:`pillarenv_from_saltenv`, and is otherwise ignored.
+
+    merge_pillar: ``True``
+        Merge the CLI variables with the pillar. Default: ``True``.
+
+    only_lower_merge: ``False``
+        Specify if it should merge only the filters and terms fields. Otherwise it will try
+        to merge everything at the policy level. Default: ``False``.
+        This option requires ``merge_pillar``, otherwise it is ignored.
+
+    revision_id
+        Add a comment in the policy config having the description for the changes applied.
+
+    revision_no
+        The revision count.
+
+    revision_date: ``True``
+        Boolean flag: display the date when the policy configuration was generated. Default: ``True``.
+
+    revision_date_format: ``%Y/%m/%d``
+        The date format to be used when generating the perforce data. Default: ``%Y/%m/%d`` (<year>/<month>/<day>).
+
+    test: ``False``
+        Dry run? If set as ``True``, will apply the config, discard and return the changes.
+        Default: ``False`` and will commit the changes on the device.
+
+    commit: ``True``
+        Commit? Default: ``True``.
+
+    debug: ``False``
+        Debug mode. Will insert a new key under the output dictionary,
+        as ``loaded_config`` contaning the raw configuration loaded on the device.
+
+    The output is a dictionary having the same form as :mod:`net.load_config <salt.modules.napalm_network.load_config>`.
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt 'edge01.flw01' netacl.load_policy_config debug=True
+
+    Output Example:
+
+    .. code-block:: yaml
+
+        edge01.flw01:
+            ----------
+            already_configured:
+                False
+            comment:
+            diff:
+                ---
+                +++
+                @@ -1228,9 +1228,24 @@
+                 !
+                +ipv4 access-list my-filter
+                + 10 remark $Id:$
+                + 20 remark my-term
+                + 30 deny tcp host 1.2.3.4 eq 1234 any
+                + 40 deny udp host 1.2.3.4 eq 1234 any
+                + 50 deny tcp host 1.2.3.4 eq 1235 any
+                + 60 deny udp host 1.2.3.4 eq 1235 any
+                + 70 remark my-other-term
+                + 80 permit tcp any range 5678 5680 any
+                +!
+                +!
+                +ipv4 access-list block-icmp
+                + 10 remark $Id:$
+                + 20 remark first-term
+                + 30 deny icmp any any
+                 !
+            loaded_config:
+                ! $Id:$
+                ! $Date:$
+                ! $Revision:$
+                no ipv4 access-list block-icmp
+                ipv4 access-list block-icmp
+                 remark $Id:$
+                 remark first-term
+                 deny icmp any any
+                exit
+                no ipv4 access-list my-filter
+                ipv4 access-list my-filter
+                 remark $Id:$
+                 remark my-term
+                 deny tcp host 1.2.3.4 eq 1234 any
+                 deny udp host 1.2.3.4 eq 1234 any
+                 deny tcp host 1.2.3.4 eq 1235 any
+                 deny udp host 1.2.3.4 eq 1235 any
+                 remark my-other-term
+                 permit tcp any range 5678 5680 any
+                exit
+            result:
+                True
+
+    The policy configuration has been loaded from the pillar, having the following structure:
+
+    .. code-block:: yaml
+
+        acl:
+          my-filter:
+            my-term:
+              source_port: [1234, 1235]
+              protocol:
+                - tcp
+                - udp
+              source_address: 1.2.3.4
+              action: reject
+            my-other-term:
+              source_port:
+                - [5678, 5680]
+              protocol: tcp
+              action: accept
+          block-icmp:
+            first-term:
+              protocol:
+                - icmp
+              action: reject
+    '''
+    if not filters:
+        filters = {}
+    platform = _get_capirca_platform()
+    policy_config = __salt__['capirca.get_policy_config'](platform,
+                                                          filters=filters,
+                                                          pillar_key=pillar_key,
+                                                          pillarenv=pillarenv,
+                                                          saltenv=saltenv,
+                                                          merge_pillar=merge_pillar,
+                                                          only_lower_merge=only_lower_merge,
+                                                          revision_id=revision_id,
+                                                          revision_no=revision_no,
+                                                          revision_date=revision_date,
+                                                          revision_date_format=revision_date_format)
+    return __salt__['net.load_config'](text=policy_config,
+                                       test=test,
+                                       commit=commit,
+                                       debug=debug,
+                                       inherit_napalm_device=napalm_device)  # pylint: disable=undefined-variable

salt/modules/saltutil.py

@@ -1151,6 +1151,13 @@ def _exec(client, tgt, fun, arg, timeout, tgt_type, ret, kwarg, **kwargs):
             'ret': ret, 'kwarg': kwarg, 'batch': kwargs['batch'],
         }
         del kwargs['batch']
+    elif 'subset' in kwargs:
+        _cmd = client.cmd_subset
+        cmd_kwargs = {
+            'tgt': tgt, 'fun': fun, 'arg': arg, 'tgt_type': tgt_type,
+            'ret': ret, 'kwarg': kwarg, 'sub': kwargs['subset'],
+        }
+        del kwargs['subset']
     else:
         _cmd = client.cmd_iter
         cmd_kwargs = {

salt/returners/librato_return.py

@@ -10,6 +10,7 @@ The librato python client can be found at:
 https://github.com/librato/python-librato
 
 .. code-block:: yaml
+
     librato.email: example@librato.com
     librato.api_token: abc12345def
 
@@ -23,6 +24,7 @@ could be modified to include the ec2 tags. Multiple dimensions are added simply
 by adding more tags to the submission.
 
 .. code-block:: python
+
     pillar_data = __salt__['pillar.raw']()
     q.add(metric.name, value, tags={'Name': ret['id'],'Region': pillar_data['ec2_tags']['Name']})
 

salt/states/netacl.py

@@ -0,0 +1,900 @@
+# -*- coding: utf-8 -*-
+'''
+Network ACL
+===========
+
+Manage the firewall configuration on the network device namaged through NAPALM.
+The firewall configuration is generated by Capirca_.
+
+.. _Capirca: https://github.com/google/capirca
+
+.. versionadded:: Nitrogen
+
+:codeauthor: Mircea Ulinic <mircea@cloudflare.com>
+:maturity:   new
+:depends:    capirca, napalm
+:platform:   unix
+
+Dependencies
+------------
+
+Capirca: ``pip install -e git+git@github.com:google/capirca.git#egg=aclgen``
+
+To be able to load configuration on network devices,
+it requires NAPALM_ library to be installed:  ``pip install napalm``.
+Please check Installation_ for complete details.
+
+.. _NAPALM: https://napalm.readthedocs.io
+.. _Installation: https://napalm.readthedocs.io/en/latest/installation.html
+'''
+from __future__ import absolute_import
+
+import logging
+log = logging.getLogger(__file__)
+
+# Import third party libs
+try:
+    # pylint: disable=W0611
+    import aclgen
+    HAS_CAPIRCA = True
+    # pylint: enable=W0611
+except ImportError:
+    HAS_CAPIRCA = False
+
+try:
+    # pylint: disable=W0611
+    import napalm_base
+    # pylint: enable=W0611
+    HAS_NAPALM = True
+except ImportError:
+    HAS_NAPALM = False
+
+# ------------------------------------------------------------------------------
+# state properties
+# ------------------------------------------------------------------------------
+
+__virtualname__ = 'netacl'
+
+# ------------------------------------------------------------------------------
+# global variables
+# ------------------------------------------------------------------------------
+
+# ------------------------------------------------------------------------------
+# property functions
+# ------------------------------------------------------------------------------
+
+
+def __virtual__():
+    '''
+    This module requires both NAPALM and Capirca.
+    '''
+    if HAS_CAPIRCA and HAS_NAPALM:
+        return __virtualname__
+    else:
+        return (False, 'The netacl state cannot be loaded: \
+                Please install capirca and napalm.')
+
+# ------------------------------------------------------------------------------
+# helper functions -- will not be exported
+# ------------------------------------------------------------------------------
+
+
+def _default_ret(name):
+    '''
+    Return the default dict of the state output.
+    '''
+    ret = {
+        'name': name,
+        'changes': {},
+        'already_configured': False,
+        'result': False,
+        'comment': ''
+    }
+    return ret
+
+
+def _loaded_ret(ret, loaded, test, debug):
+    '''
+    Return the final state output.
+
+    ret
+        The initial state output structure.
+
+    loaded
+        The loaded dictionary.
+    '''
+    applied = loaded.get('result', False)
+    result = (applied if not applied else None) if test else applied
+    _comment = loaded.get('comment', '')
+    comment = _comment if not test else 'Testing mode: {tail}'.format(tail=_comment)
+    if result is True and not comment:
+        comment = 'Configuration changed!'
+    ret.update({
+        'changes': {
+            'diff': loaded.get('diff', '')
+        },
+        'already_configured': loaded.get('already_configured', False),
+        'result': result,
+        'comment': comment,
+    })
+    if debug:
+        ret['changes']['loaded'] = loaded.get('loaded_config', '')
+    return ret
+
+# ------------------------------------------------------------------------------
+# callable functions
+# ------------------------------------------------------------------------------
+
+
+def term(name,
+         filter_name,
+         term_name,
+         filter_options=None,
+         pillar_key='acl',
+         pillarenv=None,
+         saltenv=None,
+         merge_pillar=False,
+         revision_id=None,
+         revision_no=None,
+         revision_date=True,
+         revision_date_format='%Y/%m/%d',
+         test=False,
+         commit=True,
+         debug=False,
+         source_service=None,
+         destination_service=None,
+         **term_fields):
+    '''
+    Manage the configuration of a specific policy term.
+
+    filter_name
+        The name of the policy filter.
+
+    term_name
+        The name of the term.
+
+    filter_options
+        Additional filter options. These options are platform-specific.
+        See the complete list of options_.
+
+        .. _options: https://github.com/google/capirca/wiki/Policy-format#header-section
+
+    pillar_key: ``acl``
+        The key in the pillar containing the default attributes values. Default: ``acl``.
+        If the pillar contains the following structure:
+
+    pillarenv
+        Query the master to generate fresh pillar data on the fly,
+        specifically from the requested pillar environment.
+
+    saltenv
+        Included only for compatibility with
+        :conf_minion:`pillarenv_from_saltenv`, and is otherwise ignored.
+
+    merge_pillar: ``False``
+        Merge the CLI variables with the pillar. Default: ``False``.
+
+    revision_id
+        Add a comment in the term config having the description for the changes applied.
+
+    revision_no
+        The revision count.
+
+    revision_date: ``True``
+        Boolean flag: display the date when the term configuration was generated. Default: ``True``.
+
+    revision_date_format: ``%Y/%m/%d``
+        The date format to be used when generating the perforce data. Default: ``%Y/%m/%d`` (<year>/<month>/<day>).
+
+    test: ``False``
+        Dry run? If set as ``True``, will apply the config, discard and return the changes.
+        Default: ``False`` and will commit the changes on the device.
+
+    commit: ``True``
+        Commit? Default: ``True``.
+
+    debug: ``False``
+        Debug mode. Will insert a new key under the output dictionary,
+        as ``loaded_config`` contaning the raw configuration loaded on the device.
+
+    source_service
+        A special service to choose from. This is a helper so the user is able to
+        select a source just using the name, instead of specifying a source_port and protocol.
+
+        As this module is available on Unix platforms only,
+        it reads the IANA_ port assignment from /etc/services.
+
+        If the user requires additional shortcuts to be referenced, they can add entries under /etc/services,
+        which can be managed using the :mod:`file state <salt.states.file>`.
+
+        .. _IANA: http://www.iana.org/assignments/port-numbers
+
+    destination_service
+        A special service to choose from. This is a helper so the user is able to
+        select a source just using the name, instead of specifying a destination_port and protocol.
+        Allows the same options as ``source_service``.
+
+    **term_fields
+        Term attributes.
+        To see what fields are supported, please consult the list of supported keywords_.
+            Some platforms have few other optional_ keyworkds.
+
+            .. _keywords:https://github.com/google/capirca/wiki/Policy-format#keywords
+            .. _optional: https://github.com/google/capirca/wiki/Policy-format#optionally-supported-keywords
+
+    .. note::
+        The following fields are accepted:
+
+        - action
+        - address
+        - address_exclude
+        - comment
+        - counter
+        - expiration
+        - destination_address
+        - destination_address_exclude
+        - destination_port
+        - destination_prefix
+        - forwarding_class
+        - forwarding_class_except
+        - logging
+        - log_name
+        - loss_priority
+        - option
+        - policer
+        - port
+        - precedence
+        - principals
+        - protocol
+        - protocol_except
+        - qos
+        - pan_application
+        - routing_instance
+        - source_address
+        - source_address_exclude
+        - source_port
+        - source_prefix
+        - verbatim
+        - packet_length
+        - fragment_offset
+        - hop_limit
+        - icmp_type
+        - ether_type
+        - traffic_class_count
+        - traffic_type
+        - translated
+        - dscp_set
+        - dscp_match
+        - dscp_except
+        - next_ip
+        - flexible_match_range
+        - source_prefix_except
+        - destination_prefix_except
+        - vpn
+        - source_tag
+        - destination_tag
+        - source_interface
+        - destination_interface
+        - flattened
+        - flattened_addr
+        - flattened_saddr
+        - flattened_daddr
+
+    .. note::
+        The following fields can be also a single value and a list of values:
+
+        - action
+        - address
+        - address_exclude
+        - comment
+        - destination_address
+        - destination_address_exclude
+        - destination_port
+        - destination_prefix
+        - forwarding_class
+        - forwarding_class_except
+        - logging
+        - option
+        - port
+        - precedence
+        - principals
+        - protocol
+        - protocol_except
+        - pan_application
+        - source_address
+        - source_address_exclude
+        - source_port
+        - source_prefix
+        - verbatim
+        - icmp_type
+        - ether_type
+        - traffic_type
+        - dscp_match
+        - dscp_except
+        - flexible_match_range
+        - source_prefix_except
+        - destination_prefix_except
+        - source_tag
+        - destination_tag
+        - source_service
+        - destination_service
+
+        Example: ``destination_address`` can be either defined as:
+
+        .. code-block:: yaml
+
+            destination_address: 172.17.17.1/24
+
+        or as a list of destination IP addresses:
+
+        .. code-block:: yaml
+
+            destination_address:
+                - 172.17.17.1/24
+                - 172.17.19.1/24
+
+        or a list of services to be matched:
+
+        .. code-block:: yaml
+
+            source_service:
+                - ntp
+                - snmp
+                - ldap
+                - bgpd
+
+    .. note::
+        The port fields ``source_port`` and ``destination_port`` can be used as above to select either
+        a single value, either a list of values, but also they can select port ranges. Example:
+
+        .. code-block:: yaml
+
+            source_port:
+                - [1000, 2000]
+                - [3000, 4000]
+
+        With the configuration above, the user is able to select the 1000-2000 and 3000-4000 source port ranges.
+
+    The output is a dictionary having the same form as :mod:`net.load_config <salt.modules.napalm_network.load_config>`.
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt 'edge01.bjm01' state.sls router.acl
+
+    Output Example:
+
+    .. code-block:: yaml
+
+        edge01.sfo04:
+        ----------
+                  ID: update_icmp_first_term
+            Function: netacl.term
+              Result: None
+             Comment: Testing mode: Configuration discarded.
+             Started: 12:49:09.174179
+            Duration: 5751.882 ms
+             Changes:
+                      ----------
+                      diff:
+                          [edit firewall]
+                          +    family inet {
+                          +        /*
+                          +         ** $Id: update_icmp_first_term $
+                          +         ** $Date: 2017/02/30 $
+                          +         **
+                          +         */
+                          +        filter block-icmp {
+                          +            term first-term {
+                          +                from {
+                          +                    protocol icmp;
+                          +                }
+                          +                then {
+                          +                    reject;
+                          +                }
+                          +            }
+                          +        }
+                          +    }
+
+        Summary for edge01.sfo04
+        ------------
+        Succeeded: 1 (unchanged=1, changed=1)
+        Failed:    0
+        ------------
+        Total states run:     1
+        Total run time:   5.752 s
+
+    Pillar example:
+
+    .. code-block:: yaml
+
+        firewall:
+          block-icmp:
+            first-term:
+              protocol:
+                - icmp
+              action: reject
+
+    State SLS example:
+
+    .. code-block:: yaml
+
+        {%- set filter_name = 'block-icmp' -%}
+        {%- set term_name = 'first-term' -%}
+        {%- set my_term_cfg = pillar['acl'][filter_name][term_name] -%}
+
+        update_icmp_first_term:
+          netacl.term:
+            - filter_name: {{ filter_name }}
+            - filter_options:
+                - not-interface-specific
+            - term_name: {{ term_name }}
+            - {{ my_term_cfg }}
+    '''
+    ret = _default_ret(name)
+    test = __opts__['test'] or test
+    if not filter_options:
+        filter_options = []
+    loaded = __salt__['netacl.load_term_config'](filter_name,
+                                                 term_name,
+                                                 filter_options=filter_options,
+                                                 pillar_key=pillar_key,
+                                                 pillarenv=pillarenv,
+                                                 saltenv=saltenv,
+                                                 merge_pillar=merge_pillar,
+                                                 revision_id=revision_id if revision_id else name,
+                                                 revision_no=revision_no,
+                                                 revision_date=revision_date,
+                                                 revision_date_format=revision_date_format,
+                                                 source_service=source_service,
+                                                 destination_service=destination_service,
+                                                 test=test,
+                                                 commit=commit,
+                                                 debug=debug,
+                                                 **term_fields)
+    return _loaded_ret(ret, loaded, test, debug)
+
+
+def filter(name,  # pylint: disable=redefined-builtin
+           filter_name,
+           filter_options=None,
+           terms=None,
+           pillar_key='acl',
+           pillarenv=None,
+           saltenv=None,
+           merge_pillar=False,
+           only_lower_merge=False,
+           revision_id=None,
+           revision_no=None,
+           revision_date=True,
+           revision_date_format='%Y/%m/%d',
+           test=False,
+           commit=True,
+           debug=False):
+    '''
+    Generate and load the configuration of a policy filter.
+
+    filter_name
+        The name of the policy filter.
+
+    filter_options
+        Additional filter options. These options are platform-specific.
+        See the complete list of options_.
+
+        .. _options: https://github.com/google/capirca/wiki/Policy-format#header-section
+
+    terms
+        Dictionary of terms for this policy filter.
+        If not specified or empty, will try to load the configuration from the pillar,
+        unless ``merge_pillar`` is set as ``False``.
+
+    pillar_key: ``acl``
+        The key in the pillar containing the default attributes values. Default: ``acl``.
+
+    pillarenv
+        Query the master to generate fresh pillar data on the fly,
+        specifically from the requested pillar environment.
+
+    saltenv
+        Included only for compatibility with
+        :conf_minion:`pillarenv_from_saltenv`, and is otherwise ignored.
+
+    merge_pillar: ``False``
+        Merge the CLI variables with the pillar. Default: ``True``
+
+    only_lower_merge: ``False``
+        Specify if it should merge only the terms fields. Otherwise it will try
+        to merge also filters fields. Default: ``False``.
+        This option requires ``merge_pillar``, otherwise it is ignored.
+
+    revision_id
+        Add a comment in the filter config having the description for the changes applied.
+
+    revision_no
+        The revision count.
+
+    revision_date: ``True``
+        Boolean flag: display the date when the filter configuration was generated. Default: ``True``.
+
+    revision_date_format: ``%Y/%m/%d``
+        The date format to be used when generating the perforce data. Default: ``%Y/%m/%d`` (<year>/<month>/<day>).
+
+    test: ``False``
+        Dry run? If set as ``True``, will apply the config, discard and return the changes.
+        Default: ``False`` and will commit the changes on the device.
+
+    commit: ``True``
+        Commit? Default: ``True``.
+
+    debug: ``False``
+        Debug mode. Will insert a new key under the output dictionary,
+        as ``loaded_config`` contaning the raw configuration loaded on the device.
+
+    The output is a dictionary having the same form as :mod:`net.load_config <salt.modules.napalm_network.load_config>`.
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt 'edge01.flw01' state.sls router.acl test=True
+
+    Output Example:
+
+    .. code-block:: yaml
+
+        edge01.flw01:
+        ----------
+                  ID: my-filter
+            Function: netacl.filter
+              Result: None
+             Comment: Testing mode: Configuration discarded.
+             Started: 12:24:40.598232
+            Duration: 2437.139 ms
+             Changes:
+                      ----------
+                      diff:
+                          ---
+                          +++
+                          @@ -1228,9 +1228,24 @@
+                          !
+                          +ipv4 access-list my-filter
+                          + 10 remark $Id: my-filter_state $
+                          + 20 remark $Revision: 5 $
+                          + 30 remark my-other-term
+                          + 40 permit tcp any range 5678 5680 any
+                          +!
+                          +!
+                      loaded:
+                          ! $Id: my-filter_state $
+                          ! $Revision: 5 $
+                          no ipv6 access-list my-filter
+                          ipv6 access-list my-filter
+                           remark $Id: my-filter_state $
+                           remark $Revision: 5 $
+                           remark my-other-term
+                           permit tcp any range 5678 5680 any
+                          exit
+
+        Summary for edge01.flw01
+        ------------
+        Succeeded: 1 (unchanged=1, changed=1)
+        Failed:    0
+        ------------
+        Total states run:     1
+        Total run time:   2.437 s
+
+    Pillar example:
+
+    .. code-block:: yaml
+
+        acl:
+          my-filter:
+            options:
+              - inet6
+            my-term:
+              source_port: [1234, 1235]
+              protocol:
+                - tcp
+                - udp
+              source_address: 1.2.3.4
+              action: reject
+            my-other-term:
+              source_port:
+                - [5678, 5680]
+              protocol: tcp
+              action: accept
+
+    State SLS Example:
+
+    .. code-block:: yaml
+
+        {% set my_filter_cfg = pillar.get('acl').get('my-filter') -%}
+        my-filter_state:
+          netacl.filter:
+            - filter_name: my-filter
+            - terms: {{ my_filter_cfg }}
+            - revision_date: false
+            - revision_no: 5
+            - debug: true
+
+    In the example above, as ``inet6`` has been specified in the ``filter_options``,
+    the configuration chunk referring to ``my-term`` has been ignored as it referred to
+    IPv4 only (from ``source_address`` field).
+    '''
+    ret = _default_ret(name)
+    test = __opts__['test'] or test
+    if not filter_options:
+        filter_options = []
+    if not terms:
+        terms = {}
+    loaded = __salt__['netacl.load_filter_config'](filter_name,
+                                                   filter_options=filter_options,
+                                                   terms=terms,
+                                                   pillar_key=pillar_key,
+                                                   pillarenv=pillarenv,
+                                                   saltenv=saltenv,
+                                                   merge_pillar=merge_pillar,
+                                                   only_lower_merge=only_lower_merge,
+                                                   revision_id=revision_id if revision_id else name,
+                                                   revision_no=revision_no,
+                                                   revision_date=revision_date,
+                                                   revision_date_format=revision_date_format,
+                                                   test=test,
+                                                   commit=commit,
+                                                   debug=debug)
+    return _loaded_ret(ret, loaded, test, debug)
+
+
+def managed(name,
+            filters=None,
+            pillar_key='acl',
+            pillarenv=None,
+            saltenv=None,
+            merge_pillar=False,
+            only_lower_merge=False,
+            revision_id=None,
+            revision_no=None,
+            revision_date=True,
+            revision_date_format='%Y/%m/%d',
+            test=False,
+            commit=True,
+            debug=False):
+    '''
+    Manage the whole firewall configuration.
+
+    filters
+        Dictionary of filters for this policy.
+        If not specified or empty, will try to load the configuration from the pillar,
+        unless ``merge_pillar`` is set as ``False``.
+
+    pillar_key: ``acl``
+        The key in the pillar containing the default attributes values. Default: ``acl``.
+
+    pillarenv
+        Query the master to generate fresh pillar data on the fly,
+        specifically from the requested pillar environment.
+
+    saltenv
+        Included only for compatibility with
+        :conf_minion:`pillarenv_from_saltenv`, and is otherwise ignored.
+
+    merge_pillar: ``False``
+        Merge the CLI variables with the pillar. Default: ``False``.
+
+    only_lower_merge: ``False``
+        Specify if it should merge only the filters and terms fields. Otherwise it will try
+        to merge everything at the policy level. Default: ``False``.
+        This option requires ``merge_pillar``, otherwise it is ignored.
+
+    test: ``False``
+        Dry run? If set as ``True``, will apply the config, discard and return the changes.
+        Default: ``False`` and will commit the changes on the device.
+
+    revision_id
+        Add a comment in the policy config having the description for the changes applied.
+
+    revision_no
+        The revision count.
+
+    revision_date: ``True``
+        Boolean flag: display the date when the policy configuration was generated. Default: ``True``.
+
+    revision_date_format: ``%Y/%m/%d``
+        The date format to be used when generating the perforce data. Default: ``%Y/%m/%d`` (<year>/<month>/<day>).
+
+    commit: ``True``
+        Commit? Default: ``True``.
+
+    debug: ``False``
+        Debug mode. Will insert a new key under the output dictionary,
+        as ``loaded_config`` contaning the raw configuration loaded on the device.
+
+    The output is a dictionary having the same form as :mod:`net.load_config <salt.modules.napalm_network.load_config>`.
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt 'edge01.bjm01' state.sls router.acl test=True
+
+    Output Example:
+
+    .. code-block:: yaml
+
+        edge01.bjm01:
+            ----------
+                      ID: netacl_example
+                Function: netacl.managed
+                  Result: None
+                 Comment: Testing mode: Configuration discarded.
+                 Started: 12:03:24.807023
+                Duration: 5569.453 ms
+                 Changes:
+                          ----------
+                          diff:
+                              [edit firewall]
+                              +    family inet {
+                              +        /*
+                              +         ** $Id: netacl_example $
+                              +         ** $Date: 2017/07/03 $
+                              +         ** $Revision: 2 $
+                              +         **
+                              +         */
+                              +        filter block-icmp {
+                              +            interface-specific;
+                              +            term first-term {
+                              +                from {
+                              +                    protocol icmp;
+                              +                }
+                              +                then {
+                              +                    reject;
+                              +                }
+                              +            }
+                              +        }
+                              +        /*
+                              +         ** $Id: netacl_example $
+                              +         ** $Date: 2017/07/03 $
+                              +         ** $Revision: 2 $
+                              +         **
+                              +         */
+                              +        filter my-filter {
+                              +            interface-specific;
+                              +            term my-term {
+                              +                from {
+                              +                    source-address {
+                              +                        1.2.3.4/32;
+                              +                    }
+                              +                    protocol [ tcp udp ];
+                              +                    source-port [ 1234 1235 ];
+                              +                }
+                              +                then {
+                              +                    reject;
+                              +                }
+                              +            }
+                              +            term my-other-term {
+                              +                from {
+                              +                    protocol tcp;
+                              +                    source-port 5678-5680;
+                              +                }
+                              +                then accept;
+                              +            }
+                              +        }
+                              +    }
+                          loaded:
+                              firewall {
+                                  family inet {
+                                      replace:
+                                      /*
+                                      ** $Id: netacl_example $
+                                      ** $Date: 2017/07/03 $
+                                      ** $Revision: 2 $
+                                      **
+                                      */
+                                      filter block-icmp {
+                                          interface-specific;
+                                          term first-term {
+                                              from {
+                                                  protocol icmp;
+                                              }
+                                              then {
+                                                  reject;
+                                              }
+                                          }
+                                      }
+                                  }
+                              }
+                              firewall {
+                                  family inet {
+                                      replace:
+                                      /*
+                                      ** $Id: netacl_example $
+                                      ** $Date: 2017/07/03 $
+                                      ** $Revision: 2 $
+                                      **
+                                      */
+                                      filter my-filter {
+                                          interface-specific;
+                                          term my-term {
+                                              from {
+                                                  source-address {
+                                                      1.2.3.4/32;
+                                                  }
+                                                  protocol [ tcp udp ];
+                                                  source-port [ 1234 1235 ];
+                                              }
+                                              then {
+                                                  reject;
+                                              }
+                                          }
+                                          term my-other-term {
+                                              from {
+                                                  protocol tcp;
+                                                  source-port 5678-5680;
+                                              }
+                                              then accept;
+                                          }
+                                      }
+                                  }
+                              }
+
+            Summary for edge01.bjm01
+            ------------
+            Succeeded: 1 (unchanged=1, changed=1)
+            Failed:    0
+            ------------
+            Total states run:     1
+            Total run time:   5.569 s
+
+    The policy configuration has been loaded from the pillar, having the following structure:
+
+    .. code-block:: yaml
+
+        firewall:
+          my-filter:
+            my-term:
+              source_port: [1234, 1235]
+              protocol:
+                - tcp
+                - udp
+              source_address: 1.2.3.4
+              action: reject
+            my-other-term:
+              source_port:
+                - [5678, 5680]
+              protocol: tcp
+              action: accept
+          block-icmp:
+            first-term:
+              protocol:
+                - icmp
+              action: reject
+
+    Example SLS file:
+
+    .. code-block:: yaml
+
+        {%- set fw_filters = pillar.get('firewall', {}) -%}
+        netacl_example:
+          netacl.managed:
+            - filters: {{ fw_filters }}
+            - revision_no: 2
+            - debug: true
+    '''
+    ret = _default_ret(name)
+    test = __opts__['test'] or test
+    if not filters:
+        filters = {}
+    loaded = __salt__['netacl.load_policy_config'](filters=filters,
+                                                   pillar_key=pillar_key,
+                                                   pillarenv=pillarenv,
+                                                   saltenv=saltenv,
+                                                   merge_pillar=merge_pillar,
+                                                   only_lower_merge=only_lower_merge,
+                                                   revision_id=revision_id if revision_id else name,
+                                                   revision_no=revision_no,
+                                                   revision_date=revision_date,
+                                                   revision_date_format=revision_date_format,
+                                                   test=test,
+                                                   commit=commit,
+                                                   debug=debug)
+    return _loaded_ret(ret, loaded, test, debug)

salt/states/network.py

@@ -419,7 +419,6 @@ def managed(name, type, enabled=True, **kwargs):
                         __salt__['ip.down'](name, type)
                         __salt__['ip.up'](name, type)
                         ret['changes']['status'] = 'Interface {0} restart to validate'.format(name)
-                        return ret
                 else:
                     __salt__['ip.up'](name, type)
                     ret['changes']['status'] = 'Interface {0} is up'.format(name)
@@ -433,6 +432,36 @@ def managed(name, type, enabled=True, **kwargs):
         ret['comment'] = str(error)
         return ret
 
+    # Try to enslave bonding interfaces after master was created
+    if type == 'bond' and 'noifupdown' not in kwargs:
+
+        if 'slaves' in kwargs and kwargs['slaves']:
+            # Check that there are new slaves for this master
+            present_slaves = __salt__['cmd.run'](
+                ['cat', '/sys/class/net/{0}/bonding/slaves'.format(name)]).split()
+            desired_slaves = kwargs['slaves'].split()
+            missing_slaves = set(desired_slaves) - set(present_slaves)
+
+            # Enslave only slaves missing in master
+            if missing_slaves:
+                ifenslave_path = __salt__['cmd.run'](['which', 'ifenslave']).strip()
+                if ifenslave_path:
+                    log.info("Adding slaves '{0}' to the master {1}".format(' '.join(missing_slaves), name))
+                    cmd = [ifenslave_path, name] + list(missing_slaves)
+                    __salt__['cmd.run'](cmd, python_shell=False)
+                else:
+                    log.error("Command 'ifenslave' not found")
+                ret['changes']['enslave'] = (
+                    "Added slaves '{0}' to master '{1}'"
+                    .format(' '.join(missing_slaves), name))
+            else:
+                log.info("All slaves '{0}' are already added to the master {1}"
+                         ", no actions required".format(' '.join(missing_slaves), name))
+
+    if enabled and interface_status:
+        # Interface was restarted, return
+        return ret
+
     # TODO: create saltutil.refresh_grains that fires events to the minion daemon
     grains_info = salt.loader.grains(__opts__, True)
     __grains__.update(grains_info)

salt/states/saltmod.py

@@ -79,6 +79,7 @@ def state(name,
         timeout=None,
         batch=None,
         queue=False,
+        subset=None,
         orchestration_jid=None):
     '''
     Invoke a state run on a given target
@@ -157,6 +158,11 @@ def state(name,
 
         .. versionadded:: 2016.3.0
 
+    subset
+        Number of minions from the targeted set to randomly use
+
+        .. versionadded:: Nitrogen
+
     Examples:
 
     Run a list of sls files via :py:func:`state.sls <salt.state.sls>` on target
@@ -248,6 +254,8 @@ def state(name,
 
     if batch is not None:
         cmd_kw['batch'] = str(batch)
+    if subset is not None:
+        cmd_kw['subset'] = subset
 
     masterless = __opts__['__role'] == 'minion' and \
                  __opts__['file_client'] == 'local'
@@ -367,7 +375,8 @@ def function(
         arg=None,
         kwarg=None,
         timeout=None,
-        batch=None):
+        batch=None,
+        subset=None):
     '''
     Execute a single module function on a remote minion via salt or salt-ssh
 
@@ -405,6 +414,15 @@ def function(
 
     ssh
         Set to `True` to use the ssh client instead of the standard salt client
+
+    batch
+        Execute the command :ref:`in batches <targeting-batch>`. E.g.: ``10%``.
+
+    subset
+        Number of minions from the targeted set to randomly use
+
+        .. versionadded:: Nitrogen
+
     '''
     func_ret = {'name': name,
            'changes': {},
@@ -433,6 +451,8 @@ def function(
 
     if batch is not None:
         cmd_kw['batch'] = str(batch)
+    if subset is not None:
+        cmd_kw['subset'] = subset
 
     cmd_kw['tgt_type'] = tgt_type
     cmd_kw['ssh'] = ssh

salt/utils/__init__.py

@@ -2155,7 +2155,8 @@ def namespaced_function(function, global_dict, defaults=None, preserve_context=F
         function.__code__,
         global_dict,
         name=function.__name__,
-        argdefs=defaults
+        argdefs=defaults,
+        closure=function.__closure__
     )
     new_namespaced_function.__dict__.update(function.__dict__)
     return new_namespaced_function

setup.py

@@ -4,12 +4,12 @@
 The setup script for salt
 '''
 
-from __future__ import absolute_import
-# pylint: disable=file-perms
+# pylint: disable=file-perms,ungrouped-imports,wrong-import-order,wrong-import-position,repr-flag-used-in-string
+# pylint: disable=3rd-party-local-module-not-gated
 # pylint: disable=C0111,E1101,E1103,F0401,W0611,W0201,W0232,R0201,R0902,R0903
 
 # For Python 2.5.  A no-op on 2.6 and above.
-from __future__ import print_function, with_statement
+from __future__ import absolute_import, print_function, with_statement
 
 import os
 import sys
@@ -398,12 +398,13 @@ class InstallPyCryptoWindowsWheel(Command):
         with indent_log():
             call_subprocess(call_arguments)
 
+
 def uri_to_resource(resource_file):
-    ### Returns the URI for a resource 
+    # ## Returns the URI for a resource
     # The basic case is that the resource is on saltstack.com
     # It could be the case that the resource is cached.
-    salt_uri = 'https://repo.saltstack.com/windows/dependencies/'  + resource_file
-    if os.getenv('SALTREPO_LOCAL_CACHE') == None:
+    salt_uri = 'https://repo.saltstack.com/windows/dependencies/' + resource_file
+    if os.getenv('SALTREPO_LOCAL_CACHE') is None:
         # if environment variable not set, return the basic case
         return salt_uri
     if not os.path.isdir(os.getenv('SALTREPO_LOCAL_CACHE')):
@@ -412,14 +413,14 @@ def uri_to_resource(resource_file):
     cached_resource = os.path.join(os.getenv('SALTREPO_LOCAL_CACHE'), resource_file)
     cached_resource = cached_resource.replace('/', '\\')
     if not os.path.isfile(cached_resource):
-        # if file does not exist, return the basic case    
+        # if file does not exist, return the basic case
         return salt_uri
     if os.path.getsize(cached_resource) == 0:
-        # if file has zero size, return the basic case        
-        return salt_uri            
+        # if file has zero size, return the basic case
+        return salt_uri
     return cached_resource
 
-            
+
 class InstallCompiledPyYaml(Command):
 
     description = 'Install PyYAML on Windows'
@@ -429,7 +430,6 @@ class InstallCompiledPyYaml(Command):
 
     def finalize_options(self):
         pass
-        
 
     def run(self):
         if getattr(self.distribution, 'salt_installing_pyyaml_windows', None) is None:
@@ -505,7 +505,7 @@ class DownloadWindowsDlls(Command):
                                     while True:
                                         chunk = req.read(4096)
                                         if len(chunk) == 0:
-                                            break;
+                                            break
                                         wfh.write(chunk)
                                         wfh.flush()
                                 else:
@@ -546,7 +546,7 @@ class Sdist(sdist):
             os.unlink(PACKAGED_FOR_SALT_SSH_FILE)
 
 
-class CloudSdist(Sdist):
+class CloudSdist(Sdist):  # pylint: disable=too-many-ancestors
     user_options = Sdist.user_options + [
         ('download-bootstrap-script', None,
          'Download the latest stable bootstrap-salt.sh script. This '
@@ -566,7 +566,7 @@ class CloudSdist(Sdist):
     def finalize_options(self):
         Sdist.finalize_options(self)
         if 'SKIP_BOOTSTRAP_DOWNLOAD' in os.environ:
-            log('Please stop using \'SKIP_BOOTSTRAP_DOWNLOAD\' and use '
+            log('Please stop using \'SKIP_BOOTSTRAP_DOWNLOAD\' and use '  # pylint: disable=not-callable
                 '\'DOWNLOAD_BOOTSTRAP_SCRIPT\' instead')
 
         if 'DOWNLOAD_BOOTSTRAP_SCRIPT' in os.environ:
@@ -987,7 +987,7 @@ class SaltDistribution(distutils.dist.Distribution):
                                            'virt/*.jinja',
                                            'git/*',
                                            'lxc/*',
-                                          ]}
+                                           ]}
         if not IS_WINDOWS_PLATFORM:
             package_data['salt.cloud'] = ['deploy/*.sh']
 

tests/integration/files/engines/runtests_engine.py

@@ -26,6 +26,7 @@ import salt.utils.event
 from tornado import gen
 from tornado import ioloop
 from tornado import netutil
+from tornado import iostream
 
 log = logging.getLogger(__name__)
 
@@ -114,7 +115,10 @@ class PyTestEngine(object):
         timeout = 60
         while True:
             timeout -= 1
-            event_bus.fire_event(load, master_start_event_tag, timeout=500)
-            if timeout <= 0:
+            try:
+                event_bus.fire_event(load, master_start_event_tag, timeout=500)
+                if timeout <= 0:
+                    break
+                yield gen.sleep(1)
+            except iostream.StreamClosedError:
                 break
-            yield gen.sleep(1)

tests/integration/modules/test_beacons.py

@@ -9,38 +9,33 @@ import os
 
 # Salt Libs
 from salt.exceptions import CommandExecutionError
-import tests.integration as integration
-import salt.utils
 
 # Salttesting libs
+import tests.integration as integration
 from tests.support.unit import skipIf
 
 
-BEACON_CONF_DIR = os.path.join(integration.TMP, 'minion.d')
-if not os.path.exists(BEACON_CONF_DIR):
-    os.makedirs(BEACON_CONF_DIR)
-
-IS_ADMIN = False
-if salt.utils.is_windows():
-    import salt.utils.win_functions
-    current_user = salt.utils.win_functions.get_current_user()
-    if current_user == 'SYSTEM':
-        IS_ADMIN = True
-    else:
-        IS_ADMIN = salt.utils.win_functions.is_admin(current_user)
-else:
-    IS_ADMIN = os.geteuid() == 0
-
-
 class BeaconsAddDeleteTest(integration.ModuleCase):
     '''
     Tests the add and delete functions
     '''
+    def setUp(self):
+        self.minion_conf_d_dir = os.path.join(
+                self.minion_opts['config_dir'],
+                os.path.dirname(self.minion_opts['default_include']))
+        if not os.path.isdir(self.minion_conf_d_dir):
+            os.makedirs(self.minion_conf_d_dir)
+        self.beacons_config_file_path = os.path.join(self.minion_conf_d_dir, 'beacons.conf')
+
+    def tearDown(self):
+        if os.path.isfile(self.beacons_config_file_path):
+            os.unlink(self.beacons_config_file_path)
+
     def test_add_and_delete(self):
         '''
         Test adding and deleting a beacon
         '''
-        _add = self.run_function('beacons.add', ['ps', {'apache2': 'stopped'}])
+        _add = self.run_function('beacons.add', ['ps', [{'apache2': 'stopped'}]])
         self.assertTrue(_add['result'])
 
         # save added beacon
@@ -59,10 +54,24 @@ class BeaconsTest(integration.ModuleCase):
     '''
     Tests the beacons execution module
     '''
+    beacons_config_file_path = minion_conf_d_dir = None
+
+    @classmethod
+    def tearDownClass(cls):
+        if os.path.isfile(cls.beacons_config_file_path):
+            os.unlink(cls.beacons_config_file_path)
+
     def setUp(self):
+        if self.minion_conf_d_dir is None:
+            self.minion_conf_d_dir = os.path.join(
+                    self.minion_opts['config_dir'],
+                    os.path.dirname(self.minion_opts['default_include']))
+            if not os.path.isdir(self.minion_conf_d_dir):
+                os.makedirs(self.minion_conf_d_dir)
+        self.__class__.beacons_config_file_path = os.path.join(self.minion_conf_d_dir, 'beacons.conf')
         try:
             # Add beacon to disable
-            self.run_function('beacons.add', ['ps', {'apache2': 'stopped'}])
+            self.run_function('beacons.add', ['ps', [{'apache2': 'stopped'}]])
             self.run_function('beacons.save')
         except CommandExecutionError:
             self.skipTest('Unable to add beacon')
@@ -93,7 +102,10 @@ class BeaconsTest(integration.ModuleCase):
 
         # assert beacon ps is disabled
         _list = self.run_function('beacons.list', return_yaml=False)
-        self.assertFalse(_list['ps']['enabled'])
+        for bdict in _list['ps']:
+            if 'enabled' in bdict:
+                self.assertFalse(bdict['enabled'])
+                break
 
     def test_enable(self):
         '''
@@ -131,6 +143,6 @@ class BeaconsTest(integration.ModuleCase):
         # list beacons
         ret = self.run_function('beacons.list', return_yaml=False)
         if 'enabled' in ret:
-            self.assertEqual(ret, {'ps': {'apache2': 'stopped'}, 'enabled': True})
+            self.assertEqual(ret, {'ps': [{'apache2': 'stopped'}], 'enabled': True})
         else:
             self.assertEqual(ret, {'ps': {'apache': 'stopped'}})

tests/integration/netapi/rest_tornado/test_app.py

@@ -6,13 +6,13 @@ from __future__ import print_function
 import json
 import time
 import threading
-from distutils.version import StrictVersion  # pylint: disable=import-error,no-name-in-module
 
 # Import Salt Libs
 from salt.netapi.rest_tornado import saltnado
-from tests.unit.netapi.rest_tornado.test_handlers import SaltnadoTestCase
+from salt.utils.versions import StrictVersion
 
 # Import Salt Testing Libs
+from tests.unit.netapi.rest_tornado.test_handlers import SaltnadoTestCase
 from tests.support.unit import skipIf
 
 # Import 3rd-party libs

tests/minionswarm.py

@@ -19,6 +19,7 @@ import tempfile
 import shutil
 import sys
 import hashlib
+import uuid
 
 # Import salt libs
 import salt
@@ -96,6 +97,12 @@ def parse():
         default=False,
         action='store_true',
         help='Each Minion claims a different machine id grain')
+    parser.add_option(
+        '--rand-uuid',
+        dest='rand_uuid',
+        default=False,
+        action='store_true',
+        help='Each Minion claims a different UUID grain')
     parser.add_option(
         '-k',
         '--keep-modules',
@@ -347,6 +354,8 @@ class MinionSwarm(Swarm):
             data['grains']['saltversion'] = random.choice(VERS)
         if self.opts['rand_machine_id']:
             data['grains']['machine_id'] = hashlib.md5(minion_id).hexdigest()
+        if self.opts['rand_uuid']:
+            data['grains']['uuid'] = str(uuid.uuid4())
 
         with open(path, 'w+') as fp_:
             yaml.dump(data, fp_)

tests/support/mixins.py

@@ -417,35 +417,59 @@ class _FixLoaderModuleMockMixinMroOrder(type):
 
 class LoaderModuleMockMixin(six.with_metaclass(_FixLoaderModuleMockMixinMroOrder, object)):
     def setUp(self):
-        loader_module = getattr(self, 'loader_module', None)
-        if loader_module is not None:
-            if NO_MOCK:
-                self.skipTest(NO_MOCK_REASON)
+        loader_modules = getattr(self, 'loader_module', None)
+        if loader_modules is None:
+            return
 
-            loader_module_name = loader_module.__name__
-            loader_module_globals = getattr(self, 'loader_module_globals', None)
-            loader_module_blacklisted_dunders = getattr(self, 'loader_module_blacklisted_dunders', ())
-            if loader_module_globals is None:
-                loader_module_globals = {}
-            elif callable(loader_module_globals):
-                loader_module_globals = loader_module_globals()
-            else:
-                loader_module_globals = copy.deepcopy(loader_module_globals)
+        if NO_MOCK:
+            self.skipTest(NO_MOCK_REASON)
 
-            salt_dunders = (
-                '__opts__', '__salt__', '__runner__', '__context__', '__utils__',
-                '__ext_pillar__', '__thorium__', '__states__', '__serializers__', '__ret__',
-                '__grains__', '__pillar__', '__sdb__',
-                # Proxy is commented out on purpose since some code in salt expects a NameError
-                # and is most of the time not a required dunder
-                # '__proxy__'
+        if not isinstance(loader_modules, (list, tuple)):
+            loader_modules = [loader_modules]
+
+        loader_module_globals = getattr(self, 'loader_module_globals', None)
+        loader_module_blacklisted_dunders = getattr(self, 'loader_module_blacklisted_dunders', ())
+        if loader_module_globals is None:
+            loader_module_globals = {}
+        elif callable(loader_module_globals):
+            loader_module_globals = loader_module_globals()
+        else:
+            loader_module_globals = copy.deepcopy(loader_module_globals)
+
+        minion_funcs = None
+        if '__salt__' in loader_module_globals and loader_module_globals['__salt__'] == 'autoload':
+            if '__opts__' not in loader_module_globals:
+                raise RuntimeError(
+                    'You must provide __opts__ in the loader_module_globals to auto load the minion functions'
+                )
+            import salt.loader
+            ctx = {}
+            if '__utils__' not in loader_module_globals:
+                utils = salt.loader.utils(loader_module_globals['__opts__'],
+                                          context=loader_module_globals.get('__context__') or ctx)
+                loader_module_globals['__utils__'] = utils
+            minion_funcs = salt.loader.minion_mods(
+                loader_module_globals['__opts__'],
+                context=loader_module_globals.get('__context__') or ctx,
+                utils=loader_module_globals.get('__utils__'),
             )
-            for dunder_name in salt_dunders:
-                if dunder_name not in loader_module_globals:
-                    if dunder_name in loader_module_blacklisted_dunders:
-                        continue
-                    loader_module_globals[dunder_name] = {}
+            loader_module_globals['__salt__'] = minion_funcs
 
+        salt_dunders = (
+            '__opts__', '__salt__', '__runner__', '__context__', '__utils__',
+            '__ext_pillar__', '__thorium__', '__states__', '__serializers__', '__ret__',
+            '__grains__', '__pillar__', '__sdb__',
+            # Proxy is commented out on purpose since some code in salt expects a NameError
+            # and is most of the time not a required dunder
+            # '__proxy__'
+        )
+        for dunder_name in salt_dunders:
+            if dunder_name not in loader_module_globals:
+                if dunder_name in loader_module_blacklisted_dunders:
+                    continue
+                loader_module_globals[dunder_name] = {}
+
+        for loader_module in loader_modules:
             for key in loader_module_globals:
                 if not hasattr(loader_module, key):
                     if key in salt_dunders:
@@ -454,7 +478,7 @@ class LoaderModuleMockMixin(six.with_metaclass(_FixLoaderModuleMockMixinMroOrder
                         setattr(loader_module, key, None)
 
             if loader_module_globals:
-                patcher = patch.multiple(loader_module_name, **loader_module_globals)
+                patcher = patch.multiple(loader_module, **loader_module_globals)
                 patcher.start()
 
                 def cleanup(patcher, loader_module_globals):
@@ -462,4 +486,14 @@ class LoaderModuleMockMixin(six.with_metaclass(_FixLoaderModuleMockMixinMroOrder
                     del loader_module_globals
 
                 self.addCleanup(cleanup, patcher, loader_module_globals)
+        if minion_funcs is not None:
+            # Since we autoloaded the minion_funcs, let's namespace the functions with the globals
+            # used to patch above
+            import salt.utils
+            for func in minion_funcs:
+                minion_funcs[func] = salt.utils.namespaced_function(
+                    minion_funcs[func],
+                    loader_module_globals,
+                    preserve_context=True
+                )
         super(LoaderModuleMockMixin, self).setUp()

tests/support/xmlunit.py

@@ -10,15 +10,16 @@
 
     XML Unit Tests
 '''
+# pylint: disable=wrong-import-order,wrong-import-position
 
 # Import python libs
 from __future__ import absolute_import
+import io
 import sys
 import logging
 
 # Import 3rd-party libs
 import salt.ext.six as six
-from salt.ext.six.moves import StringIO  # pylint: disable=import-error
 
 log = logging.getLogger(__name__)
 
@@ -35,7 +36,7 @@ try:
         '''
 
         def __init__(self, delegate):
-            self._captured = StringIO()
+            self._captured = six.StringIO()
             self.delegate = delegate
 
         def write(self, text):
@@ -44,17 +45,18 @@ try:
             self._captured.write(text)
             self.delegate.write(text)
 
+        def fileno(self):
+            return self.delegate.fileno()
+
         def __getattr__(self, attr):
             try:
                 return getattr(self._captured, attr)
-            except AttributeError:
+            except (AttributeError, io.UnsupportedOperation):
                 return getattr(self.delegate, attr)
 
     class _XMLTestResult(xmlrunner.result._XMLTestResult):
         def startTest(self, test):
-            logging.getLogger(__name__).debug(
-                '>>>>> START >>>>> {0}'.format(test.id())
-            )
+            log.debug('>>>>> START >>>>> {0}'.format(test.id()))
             # xmlrunner classes are NOT new-style classes
             xmlrunner.result._XMLTestResult.startTest(self, test)
             if self.buffer:
@@ -66,9 +68,7 @@ try:
                 sys.stdout = self._stdout_buffer
 
         def stopTest(self, test):
-            logging.getLogger(__name__).debug(
-                '<<<<< END <<<<<<< {0}'.format(test.id())
-            )
+            log.debug('<<<<< END <<<<<<< {0}'.format(test.id()))
             # xmlrunner classes are NOT new-style classes
             return xmlrunner.result._XMLTestResult.stopTest(self, test)
 

tests/unit/beacons/test_inotify_beacon.py

@@ -11,8 +11,6 @@ from salt.beacons import inotify
 
 # Salt testing libs
 from tests.support.unit import skipIf, TestCase
-from tests.support.helpers import destructiveTest
-from tests.support.mock import NO_MOCK, NO_MOCK_REASON
 
 # Third-party libs
 try:
@@ -23,20 +21,23 @@ except ImportError:
 
 
 @skipIf(not HAS_PYINOTIFY, 'pyinotify is not available')
-@skipIf(NO_MOCK, NO_MOCK_REASON)
 class INotifyBeaconTestCase(TestCase):
     '''
     Test case for salt.beacons.inotify
     '''
     def setUp(self):
         inotify.__context__ = {}
+        self.tmpdir = tempfile.mkdtemp()
 
-    def test_empty_config(self, *args, **kwargs):
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir, ignore_errors=True)
+
+    def test_empty_config(self):
         config = {}
         ret = inotify.beacon(config)
         self.assertEqual(ret, [])
 
-    def test_file_open(self, *args, **kwargs):
+    def test_file_open(self):
         path = os.path.realpath(__file__)
         config = {path: {'mask': ['open']}}
         ret = inotify.beacon(config)
@@ -49,119 +50,87 @@ class INotifyBeaconTestCase(TestCase):
         self.assertEqual(ret[0]['path'], path)
         self.assertEqual(ret[0]['change'], 'IN_OPEN')
 
-    @destructiveTest
-    def test_dir_no_auto_add(self, *args, **kwargs):
-        tmpdir = None
-        try:
-            tmpdir = tempfile.mkdtemp()
-            config = {tmpdir: {'mask': ['create']}}
-            ret = inotify.beacon(config)
-            self.assertEqual(ret, [])
-            fp = os.path.join(tmpdir, 'tmpfile')
-            with open(fp, 'w') as f:
-                pass
-            ret = inotify.beacon(config)
-            self.assertEqual(len(ret), 1)
-            self.assertEqual(ret[0]['path'], fp)
-            self.assertEqual(ret[0]['change'], 'IN_CREATE')
-            with open(fp, 'r') as f:
-                pass
-            ret = inotify.beacon(config)
-            self.assertEqual(ret, [])
+    def test_dir_no_auto_add(self):
+        config = {self.tmpdir: {'mask': ['create']}}
+        ret = inotify.beacon(config)
+        self.assertEqual(ret, [])
+        fp = os.path.join(self.tmpdir, 'tmpfile')
+        with open(fp, 'w') as f:
+            pass
+        ret = inotify.beacon(config)
+        self.assertEqual(len(ret), 1)
+        self.assertEqual(ret[0]['path'], fp)
+        self.assertEqual(ret[0]['change'], 'IN_CREATE')
+        with open(fp, 'r') as f:
+            pass
+        ret = inotify.beacon(config)
+        self.assertEqual(ret, [])
 
-        finally:
-            if tmpdir:
-                shutil.rmtree(tmpdir)
+    def test_dir_auto_add(self):
+        config = {self.tmpdir: {'mask': ['create', 'open'], 'auto_add': True}}
+        ret = inotify.beacon(config)
+        self.assertEqual(ret, [])
+        fp = os.path.join(self.tmpdir, 'tmpfile')
+        with open(fp, 'w') as f:
+            pass
+        ret = inotify.beacon(config)
+        self.assertEqual(len(ret), 2)
+        self.assertEqual(ret[0]['path'], fp)
+        self.assertEqual(ret[0]['change'], 'IN_CREATE')
+        self.assertEqual(ret[1]['path'], fp)
+        self.assertEqual(ret[1]['change'], 'IN_OPEN')
+        with open(fp, 'r') as f:
+            pass
+        ret = inotify.beacon(config)
+        self.assertEqual(len(ret), 1)
+        self.assertEqual(ret[0]['path'], fp)
+        self.assertEqual(ret[0]['change'], 'IN_OPEN')
 
-    @destructiveTest
-    def test_dir_auto_add(self, *args, **kwargs):
-        tmpdir = None
-        try:
-            tmpdir = tempfile.mkdtemp()
-            config = {tmpdir: {'mask': ['create', 'open'], 'auto_add': True}}
-            ret = inotify.beacon(config)
-            self.assertEqual(ret, [])
-            fp = os.path.join(tmpdir, 'tmpfile')
-            with open(fp, 'w') as f:
-                pass
-            ret = inotify.beacon(config)
-            self.assertEqual(len(ret), 2)
-            self.assertEqual(ret[0]['path'], fp)
-            self.assertEqual(ret[0]['change'], 'IN_CREATE')
-            self.assertEqual(ret[1]['path'], fp)
-            self.assertEqual(ret[1]['change'], 'IN_OPEN')
-            with open(fp, 'r') as f:
-                pass
-            ret = inotify.beacon(config)
-            self.assertEqual(len(ret), 1)
-            self.assertEqual(ret[0]['path'], fp)
-            self.assertEqual(ret[0]['change'], 'IN_OPEN')
+    def test_dir_recurse(self):
+        dp1 = os.path.join(self.tmpdir, 'subdir1')
+        os.mkdir(dp1)
+        dp2 = os.path.join(dp1, 'subdir2')
+        os.mkdir(dp2)
+        fp = os.path.join(dp2, 'tmpfile')
+        with open(fp, 'w') as f:
+            pass
+        config = {self.tmpdir: {'mask': ['open'], 'recurse': True}}
+        ret = inotify.beacon(config)
+        self.assertEqual(ret, [])
+        with open(fp) as f:
+            pass
+        ret = inotify.beacon(config)
+        self.assertEqual(len(ret), 3)
+        self.assertEqual(ret[0]['path'], dp1)
+        self.assertEqual(ret[0]['change'], 'IN_OPEN|IN_ISDIR')
+        self.assertEqual(ret[1]['path'], dp2)
+        self.assertEqual(ret[1]['change'], 'IN_OPEN|IN_ISDIR')
+        self.assertEqual(ret[2]['path'], fp)
+        self.assertEqual(ret[2]['change'], 'IN_OPEN')
 
-        finally:
-            if tmpdir:
-                shutil.rmtree(tmpdir)
-
-    @destructiveTest
-    def test_dir_recurse(self, *args, **kwargs):
-        tmpdir = None
-        try:
-            tmpdir = tempfile.mkdtemp()
-            dp1 = os.path.join(tmpdir, 'subdir1')
-            os.mkdir(dp1)
-            dp2 = os.path.join(dp1, 'subdir2')
-            os.mkdir(dp2)
-            fp = os.path.join(dp2, 'tmpfile')
-            with open(fp, 'w') as f:
-                pass
-            config = {tmpdir: {'mask': ['open'], 'recurse': True}}
-            ret = inotify.beacon(config)
-            self.assertEqual(ret, [])
-            with open(fp) as f:
-                pass
-            ret = inotify.beacon(config)
-            self.assertEqual(len(ret), 3)
-            self.assertEqual(ret[0]['path'], dp1)
-            self.assertEqual(ret[0]['change'], 'IN_OPEN|IN_ISDIR')
-            self.assertEqual(ret[1]['path'], dp2)
-            self.assertEqual(ret[1]['change'], 'IN_OPEN|IN_ISDIR')
-            self.assertEqual(ret[2]['path'], fp)
-            self.assertEqual(ret[2]['change'], 'IN_OPEN')
-
-        finally:
-            if tmpdir:
-                shutil.rmtree(tmpdir)
-
-    @destructiveTest
-    def test_dir_recurse_auto_add(self, *args, **kwargs):
-        tmpdir = None
-        try:
-            tmpdir = tempfile.mkdtemp()
-            dp1 = os.path.join(tmpdir, 'subdir1')
-            os.mkdir(dp1)
-            config = {tmpdir: {'mask': ['create', 'delete'],
-                               'recurse': True,
-                               'auto_add': True}}
-            ret = inotify.beacon(config)
-            self.assertEqual(ret, [])
-            dp2 = os.path.join(dp1, 'subdir2')
-            os.mkdir(dp2)
-            ret = inotify.beacon(config)
-            self.assertEqual(len(ret), 1)
-            self.assertEqual(ret[0]['path'], dp2)
-            self.assertEqual(ret[0]['change'], 'IN_CREATE|IN_ISDIR')
-            fp = os.path.join(dp2, 'tmpfile')
-            with open(fp, 'w') as f:
-                pass
-            ret = inotify.beacon(config)
-            self.assertEqual(len(ret), 1)
-            self.assertEqual(ret[0]['path'], fp)
-            self.assertEqual(ret[0]['change'], 'IN_CREATE')
-            os.remove(fp)
-            ret = inotify.beacon(config)
-            self.assertEqual(len(ret), 1)
-            self.assertEqual(ret[0]['path'], fp)
-            self.assertEqual(ret[0]['change'], 'IN_DELETE')
-
-        finally:
-            if tmpdir:
-                shutil.rmtree(tmpdir)
+    def test_dir_recurse_auto_add(self):
+        dp1 = os.path.join(self.tmpdir, 'subdir1')
+        os.mkdir(dp1)
+        config = {self.tmpdir: {'mask': ['create', 'delete'],
+                                'recurse': True,
+                                'auto_add': True}}
+        ret = inotify.beacon(config)
+        self.assertEqual(ret, [])
+        dp2 = os.path.join(dp1, 'subdir2')
+        os.mkdir(dp2)
+        ret = inotify.beacon(config)
+        self.assertEqual(len(ret), 1)
+        self.assertEqual(ret[0]['path'], dp2)
+        self.assertEqual(ret[0]['change'], 'IN_CREATE|IN_ISDIR')
+        fp = os.path.join(dp2, 'tmpfile')
+        with open(fp, 'w') as f:
+            pass
+        ret = inotify.beacon(config)
+        self.assertEqual(len(ret), 1)
+        self.assertEqual(ret[0]['path'], fp)
+        self.assertEqual(ret[0]['change'], 'IN_CREATE')
+        os.remove(fp)
+        ret = inotify.beacon(config)
+        self.assertEqual(len(ret), 1)
+        self.assertEqual(ret[0]['path'], fp)
+        self.assertEqual(ret[0]['change'], 'IN_DELETE')

tests/unit/beacons/test_status.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+'''
+    :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)`
+    :copyright: © 2017 by the SaltStack Team, see AUTHORS for more details.
+
+
+    tests.unit.beacons.test_status
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Status beacon test cases
+'''
+
+# Python libs
+from __future__ import absolute_import
+
+# Salt libs
+import salt.config
+import salt.loader
+from salt.beacons import status
+from salt.modules import status as status_module
+
+# Salt testing libs
+from tests.support.unit import TestCase
+from tests.support.mixins import LoaderModuleMockMixin
+
+
+class StatusBeaconTestCase(TestCase, LoaderModuleMockMixin):
+    '''
+    Test case for salt.beacons.status
+    '''
+    loader_module = (status, status_module)
+
+    def loader_module_globals(self):
+        opts = salt.config.DEFAULT_MINION_OPTS
+        return {
+            '__opts__': opts,
+            '__salt__': 'autoload',
+            '__context__': {},
+            '__grains__': {'kernel': 'Linux'}
+        }
+
+    def test_empty_config(self, *args, **kwargs):
+        config = {}
+        ret = status.beacon(config)
+        self.assertEqual(ret[0]['data'].keys(), ['loadavg', 'meminfo', 'cpustats', 'vmstats', 'time'])
+
+    def test_deprecated_dict_config(self):
+        config = {'time': ['all']}
+        ret = status.beacon(config)
+        self.assertEqual(ret[0]['data'].keys(), ['time'])
+
+    def test_list_config(self):
+        config = [{'time': ['all']}]
+        ret = status.beacon(config)
+        self.assertEqual(ret[0]['data'].keys(), ['time'])