Merge pull request #27525 from basepi/merge-forward-2015.8

[2015.8] Merge forward from 2015.5 to 2015.8

doc/_themes/saltstack2/layout.html

@@ -26,7 +26,7 @@
 
 {% set css_files = [
     '_static/css/core.min.css',
-    '_static/css/webhelp.min_v1.4.2.css',
+    '_static/css/webhelp.min_v1.4.3.css',
 ] %}
 
 {%- macro relbar() %}

doc/_themes/saltstack2/static/css/webhelp.min_v1.4.3.css

@@ -311,6 +311,8 @@ dl{margin-bottom:15px}
 dd p{margin-top:0}
 dd ul,dd table{margin-bottom:10px}
 dd{margin-top:3px;margin-bottom:10px;margin-left:30px}
+dd table{table-layout:fixed;width:100%}
+dd table th.field-name{width:20%}
 dl.glossary dt{font-weight:700;font-size:1.1em}
 .field-list ul{margin:0;padding-left:1em}
 .field-list p{margin:0}

doc/ref/file_server/dynamic-modules.rst

@@ -1,23 +1,30 @@
 .. _module-sync:
 
 ===========================
+.. _dynamic-module-distribution:
+
 Dynamic Module Distribution
 ===========================
 
 .. versionadded:: 0.9.5
 
-Salt Python modules can be distributed automatically via the Salt file server.
+Custom Salt execution, state, and other modules can be distributed to Salt
+minions using the Salt file server.
+
 Under the root of any environment defined via the :conf_master:`file_roots`
 option on the master server directories corresponding to the type of module can
 be used.
 
 The directories are prepended with an underscore:
 
-  1. :file:`_modules`
-  2. :file:`_grains`
-  3. :file:`_renderers`
-  4. :file:`_returners`
-  5. :file:`_states`
+- :file:`_beacons`
+- :file:`_modules`
+- :file:`_grains`
+- :file:`_renderers`
+- :file:`_returners`
+- :file:`_states`
+- :file:`_output`
+- :file:`_utils`
 
 The contents of these directories need to be synced over to the minions after
 Python modules have been created in them. There are a number of ways to sync
@@ -27,9 +34,9 @@ Sync Via States
 ===============
 
 The minion configuration contains an option ``autoload_dynamic_modules``
-which defaults to True. This option makes the state system refresh all
+which defaults to ``True``. This option makes the state system refresh all
 dynamic modules when states are run. To disable this behavior set
-``autoload_dynamic_modules`` to False in the minion config.
+:conf_minion:`autoload_dynamic_modules` to ``False`` in the minion config.
 
 When dynamic modules are autoloaded via states, modules only pertinent to
 the environments matched in the master's top file are downloaded.

doc/topics/cloud/softlayer.rst

@@ -278,8 +278,8 @@ Set up an initial profile at ``/etc/salt/cloud.profiles``:
       image: 13963
       # 2 x 2.0 GHz Core Bare Metal Instance - 2 GB Ram
       size: 1921
-      # 250GB SATA II
-      hdd: 19
+      # 500GB SATA II
+      hdd: 1267
       # San Jose 01
       location: 168642
       domain: example.com
@@ -318,15 +318,14 @@ contain. The `id` will be the setting to be used in the profile.
 
 hdd
 ---
-There are currently two sizes of hard disk drive (HDD) that are available for
+There is currently only one size of hard disk drive (HDD) that is available for
 hardware instances on SoftLayer:
 
 .. code-block:: yaml
 
-    19: 250GB SATA II
     1267: 500GB SATA II
 
-The `hdd` setting in the profile will be either 19 or 1267. Other sizes may be
+The `hdd` setting in the profile should be 1267. Other sizes may be
 added in the future.
 
 location
@@ -444,8 +443,8 @@ them, that can be passed into Salt Cloud with the `optional_products` option:
       image: 13963
       # 2 x 2.0 GHz Core Bare Metal Instance - 2 GB Ram
       size: 1921
-      # 250GB SATA II
-      hdd: 19
+      # 500GB SATA II
+      hdd: 1267
       # San Jose 01
       location: 168642
       domain: example.com

doc/topics/ssh/index.rst

@@ -60,6 +60,42 @@ address. A more elaborate roster can be created:
     sudo works only if NOPASSWD is set for user in /etc/sudoers:
     ``fred ALL=(ALL) NOPASSWD: ALL``
 
+Deploy ssh key for salt-ssh
+===========================
+
+By default, salt-ssh will generate key pairs for ssh, the default path will be
+/etc/salt/pki/master/ssh/salt-ssh.rsa
+
+You can use ssh-copy-id, (the OpenSSH key deployment tool) to deploy keys to your servers.
+
+.. code-block:: bash
+
+   ssh-copy-id -i /etc/salt/pki/master/ssh/salt-ssh.rsa user@server.demo.com
+
+One could also create e a simple shell script, named salt-ssh-copy-id.sh as follows:
+
+.. code-block:: bash
+
+   #!/bin/bash
+   if [ -z $1 ]; then
+      echo $0 user@host.com
+      exit 0
+   fi
+   ssh-copy-id -i /etc/salt/pki/master/ssh/salt-ssh.rsa $1
+
+
+.. note::
+
+Be certain to chmod +x salt-ssh-copy-id.sh.
+
+.. code-block:: bash
+
+   ./salt-ssh-copy-id.sh user@server1.host.com
+   ./salt-ssh-copy-id.sh user@server2.host.com
+
+Once keys are successfully deployed, salt-ssh can be used to control them.
+
+
 Calling Salt SSH
 ================
 

doc/topics/windows/windows-package-manager.rst

@@ -71,30 +71,30 @@ The package definition file should look similar to this example for Firefox:
 .. code-block:: yaml
 
     firefox:
-      17.0.1:
+      '17.0.1':
         installer: 'salt://win/repo/firefox/English/Firefox Setup 17.0.1.exe'
         full_name: Mozilla Firefox 17.0.1 (x86 en-US)
         locale: en_US
         reboot: False
-        install_flags: ' -ms'
+        install_flags: '-ms'
         uninstaller: '%ProgramFiles(x86)%/Mozilla Firefox/uninstall/helper.exe'
-        uninstall_flags: ' /S'
-      16.0.2:
+        uninstall_flags: '/S'
+      '16.0.2':
         installer: 'salt://win/repo/firefox/English/Firefox Setup 16.0.2.exe'
         full_name: Mozilla Firefox 16.0.2 (x86 en-US)
         locale: en_US
         reboot: False
-        install_flags: ' -ms'
+        install_flags: '-ms'
         uninstaller: '%ProgramFiles(x86)%/Mozilla Firefox/uninstall/helper.exe'
-        uninstall_flags: ' /S'
-      15.0.1:
+        uninstall_flags: '/S'
+      '15.0.1':
         installer: 'salt://win/repo/firefox/English/Firefox Setup 15.0.1.exe'
         full_name: Mozilla Firefox 15.0.1 (x86 en-US)
         locale: en_US
         reboot: False
-        install_flags: ' -ms'
+        install_flags: '-ms'
         uninstaller: '%ProgramFiles(x86)%/Mozilla Firefox/uninstall/helper.exe'
-        uninstall_flags: ' /S'
+        uninstall_flags: '/S'
 
 More examples can be found here: https://github.com/saltstack/salt-winrepo
 

salt/client/ssh/state.py

@@ -136,7 +136,7 @@ def prep_trans_tar(file_client, chunks, file_refs, pillar=None):
             [salt.utils.url.create('_grains')],
             [salt.utils.url.create('_renderers')],
             [salt.utils.url.create('_returners')],
-            [salt.utils.url.create('_outputters')],
+            [salt.utils.url.create('_output')],
             [salt.utils.url.create('_utils')],
             ]
     with salt.utils.fopen(lowfn, 'w+') as fp_:

salt/modules/darwin_sysctl.py

@@ -176,9 +176,11 @@ def persist(name, value, config='/etc/sysctl.conf', apply_change=False):
                     return 'Already set'
                 new_line = '{0}={1}'.format(name, value)
                 nlines.append(new_line)
+                nlines.append('\n')
                 edited = True
     if not edited:
         nlines.append('{0}={1}'.format(name, value))
+        nlines.append('\n')
     with salt.utils.fopen(config, 'w+') as ofile:
         ofile.writelines(nlines)
     # If apply_change=True, apply edits to system

salt/modules/defaults.py

@@ -1,18 +1,20 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
-import inspect
 import json
+import logging
 import os
 import yaml
 
 import salt.fileclient
 import salt.utils
 import salt.utils.url
-import salt.ext.six as six
 
 __virtualname__ = 'defaults'
 
 
+log = logging.getLogger(__name__)
+
+
 def _mk_client():
     '''
     Create a file client and add it to the context
@@ -22,34 +24,44 @@ def _mk_client():
             salt.fileclient.get_file_client(__opts__)
 
 
-def _get_files(pillar_name):
+def _load(formula):
     '''
-    Generates a list of salt://<pillar_name>/defaults.(json|yaml) files
+    Generates a list of salt://<formula>/defaults.(json|yaml) files
     and fetches them from the Salt master.
+
+    Returns first defaults file as python dict.
     '''
+
+    # Compute possibilities
     _mk_client()
-    pillar_name = pillar_name.replace('.', '/')
     paths = []
-
     for ext in ('yaml', 'json'):
-        source_url = salt.utils.url.create(pillar_name + '/defaults.' + ext)
+        source_url = salt.utils.url.create(formula + '/defaults.' + ext)
         paths.append(source_url)
+    # Fetch files from master
+    defaults_files = __context__['cp.fileclient'].cache_files(paths)
 
-    return __context__['cp.fileclient'].cache_files(paths)
+    for file_ in defaults_files:
+        if not file_:
+            # Skip empty string returned by cp.fileclient.cache_files.
+            continue
 
+        suffix = file_.rsplit('.', 1)[-1]
+        if suffix == 'yaml':
+            loader = yaml
+        elif suffix == 'json':
+            loader = json
+        else:
+            log.debug("Failed to determine loader for %r", file_)
+            continue
 
-def _load(defaults_path):
-    '''
-    Given a pillar_name and the template cache location, attempt to load
-    the defaults.json from the cache location. If it does not exist, try
-    defaults.yaml.
-    '''
-    for loader in json, yaml:
-        defaults_file = os.path.join(defaults_path, 'defaults.' + loader.__name__)
-        if os.path.exists(defaults_file):
-            with salt.utils.fopen(defaults_file) as fhr:
+        if os.path.exists(file_):
+            log.debug("Reading defaults from %r", file_)
+            with salt.utils.fopen(file_) as fhr:
                 defaults = loader.load(fhr)
-            return defaults
+                log.debug("Read defaults %r", defaults)
+
+            return defaults or {}
 
 
 def get(key, default=''):
@@ -58,79 +70,30 @@ def get(key, default=''):
     a default value for a pillar from defaults.json or defaults.yaml
     files that are stored in the root of a salt formula.
 
-    When called from the CLI it works exactly like pillar.get.
-
     CLI Example:
 
     .. code-block:: bash
 
         salt '*' defaults.get core:users:root
 
-    When called from an SLS file, it works by first reading a defaults.json
-    and second a defaults.yaml file. If the key exists in these files and
-    does not exist in a pillar named after the formula, the value from the
-    defaults file is used.
+    The defaults is computed from pillar key. The first entry is considered as
+    the formula namespace.
 
-    Example core/defaults.json file for the 'core' formula:
-
-    .. code-block:: json
-
-        {
-            "users": {
-                "root": 0
-            }
-        }
-
-    With this, from a state file you can use salt['defaults.get']('users:root')
-    to read the '0' value from defaults.json if a core:users:root pillar
-    key is not defined.
+    For example, querying ``core:users:root`` will try to load
+    ``salt://core/defaults.yaml`` and ``salt://core/defaults.json``.
     '''
 
-    sls = None
-    tmplpath = None
+    # Determine formula namespace from query
+    if ':' in key:
+        namespace, key = key.split(':', 1)
+    else:
+        namespace, key = key, None
 
-    for frame in inspect.stack():
-        if sls is not None and tmplpath is not None:
-            break
+    # Fetch and load defaults formula files from states.
+    defaults = _load(namespace)
 
-        frame_args = inspect.getargvalues(frame[0]).locals
-
-        for _sls in (
-            None if not isinstance(frame_args.get('context'), dict) else frame_args.get('context').get('__sls__'),
-            frame_args.get('mods', [None])[0],
-            frame_args.get('sls')
-        ):
-            if sls is not None:
-                break
-            sls = _sls
-
-        for _tmpl in (
-            frame_args.get('tmplpath'),
-            frame_args.get('tmplsrc')
-        ):
-            if tmplpath is not None:
-                break
-            tmplpath = _tmpl
-
-    if not sls:  # this is the case when called from CLI
-        return __salt__['pillar.get'](key, default)
-
-    pillar_name = sls.split('.')[0]
-    defaults_path = tmplpath.split(pillar_name)[0] + pillar_name
-
-    _get_files(pillar_name)
-
-    defaults = _load(defaults_path)
-
-    value = __salt__['pillar.get']('{0}:{1}'.format(pillar_name, key), None)
-
-    if value is None:
-        value = salt.utils.traverse_dict_and_list(defaults, key, None)
-
-    if value is None:
-        value = default
-
-    if isinstance(value, six.text_type):
-        value = str(value)
-
-    return value
+    # Fetch value
+    if key:
+        return salt.utils.traverse_dict_and_list(defaults, key, default)
+    else:
+        return defaults

salt/modules/file.py

@@ -1658,7 +1658,9 @@ def replace(path,
     repl
         The replacement text
     count
-        Maximum number of pattern occurrences to be replaced
+        Maximum number of pattern occurrences to be replaced.  Defaults to 0.
+        If count is a positive integer n, only n occurrences will be replaced,
+        otherwise all occurrences will be replaced.
     flags (list or int)
         A list of flags defined in the :ref:`re module documentation
         <contents-of-module-re>`. Each list item should be a string that will
@@ -1823,7 +1825,7 @@ def replace(path,
                 if prepend_if_not_found or append_if_not_found:
                     # Search for content, so we don't continue pre/appending
                     # the content if it's been pre/appended in a previous run.
-                    if re.search('^{0}$'.format(content), line):
+                    if re.search('^{0}$'.format(re.escape(content)), line):
                         # Content was found, so set found.
                         found = True
 

salt/modules/localemod.py

@@ -125,6 +125,8 @@ def get_locale():
         return _locale_get()
     elif 'RedHat' in __grains__['os_family']:
         cmd = 'grep "^LANG=" /etc/sysconfig/i18n'
+    elif 'Suse' in __grains__['os_family']:
+        cmd = 'grep "^RC_LANG" /etc/sysconfig/language'
     elif 'Debian' in __grains__['os_family']:
         cmd = 'grep "^LANG=" /etc/default/locale'
     elif 'Gentoo' in __grains__['os_family']:
@@ -158,6 +160,15 @@ def set_locale(locale):
             'LANG="{0}"'.format(locale),
             append_if_not_found=True
         )
+    elif 'Suse' in __grains__['os_family']:
+        if not __salt__['file.file_exists']('/etc/sysconfig/language'):
+            __salt__['file.touch']('/etc/sysconfig/language')
+        __salt__['file.replace'](
+            '/etc/sysconfig/language',
+            '^RC_LANG=.*',
+            'RC_LANG="{0}"'.format(locale),
+            append_if_not_found=True
+        )
     elif 'Debian' in __grains__['os_family']:
         update_locale = salt.utils.which('update-locale')
         if update_locale is None:

salt/modules/mysql.py

@@ -1612,7 +1612,8 @@ def grant_exists(grant,
     grants = user_grants(user, host, **connection_args)
 
     if grants is False:
-        log.debug('Grant does not exist, or is perhaps not ordered properly?')
+        log.error('Grant does not exist or may not be ordered properly. In some cases, '
+                  'this could also indicate a connection error. Check your configuration.')
         return False
 
     target_tokens = None

salt/modules/saltutil.py

@@ -390,24 +390,26 @@ def sync_returners(saltenv=None, refresh=True):
     return ret
 
 
-def sync_outputters(saltenv=None, refresh=True):
+def sync_output(saltenv=None, refresh=True):
     '''
-    Sync the outputters from the _outputters directory on the salt master file
-    server. This function is environment aware, pass the desired environment
-    to grab the contents of the _outputters directory, base is the default
+    Sync the output modules from the _output directory on the salt master file
+    server. This function is environment aware. Pass the desired environment
+    to grab the contents of the _output directory. Base is the default
     environment.
 
     CLI Example:
 
     .. code-block:: bash
 
-        salt '*' saltutil.sync_outputters
+        salt '*' saltutil.sync_output
     '''
-    ret = _sync('outputters', saltenv)
+    ret = _sync('output', saltenv)
     if refresh:
         refresh_modules()
     return ret
 
+sync_outputters = sync_output
+
 
 def sync_utils(saltenv=None, refresh=True):
     '''
@@ -453,7 +455,7 @@ def sync_all(saltenv=None, refresh=True):
     '''
     Sync down all of the dynamic modules from the file server for a specific
     environment. This function synchronizes custom modules, states, beacons,
-    grains, returners, outputters, renderers, and utils.
+    grains, returners, output modules, renderers, and utils.
 
     refresh : True
         Also refresh the execution modules available to the minion.
@@ -489,7 +491,7 @@ def sync_all(saltenv=None, refresh=True):
     ret['grains'] = sync_grains(saltenv, False)
     ret['renderers'] = sync_renderers(saltenv, False)
     ret['returners'] = sync_returners(saltenv, False)
-    ret['outputters'] = sync_outputters(saltenv, False)
+    ret['output'] = sync_output(saltenv, False)
     ret['utils'] = sync_utils(saltenv, False)
     ret['log_handlers'] = sync_log_handlers(saltenv, False)
     if refresh:

salt/returners/mysql.py

@@ -92,7 +92,7 @@ Use the following mysql database schema:
     CREATE TABLE `salt_events` (
     `id` BIGINT NOT NULL AUTO_INCREMENT,
     `tag` varchar(255) NOT NULL,
-    `data` varchar(1024) NOT NULL,
+    `data` mediumtext NOT NULL,
     `alter_time` TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     `master_id` varchar(255) NOT NULL,
     PRIMARY KEY (`id`),

salt/state.py

@@ -2540,7 +2540,11 @@ class BaseHighState(object):
         '''
         Returns the high data derived from the top file
         '''
-        tops = self.get_tops()
+        try:
+            tops = self.get_tops()
+        except SaltRenderError as err:
+            log.error('Unable to render top file: ' + str(err.error))
+            return {}
         return self.merge_tops(tops)
 
     def top_matches(self, top):
@@ -3053,7 +3057,7 @@ class BaseHighState(object):
             top = self.get_top()
         except SaltRenderError as err:
             ret[tag_name]['comment'] = 'Unable to render top file: '
-            ret[tag_name]['comment'] += err.error
+            ret[tag_name]['comment'] += str(err.error)
             return ret
         except Exception:
             trb = traceback.format_exc()
@@ -3062,7 +3066,7 @@ class BaseHighState(object):
         err += self.verify_tops(top)
         matches = self.top_matches(top)
         if not matches:
-            msg = ('No Top file or external nodes data matches found')
+            msg = 'No Top file or external nodes data matches found.'
             ret[tag_name]['comment'] = msg
             return ret
         matches = self.matches_whitelist(matches, whitelist)

salt/states/archive.py

@@ -208,7 +208,7 @@ def extracted(name,
 
     log.debug('Extract {0} in {1}'.format(filename, name))
     if archive_format == 'zip':
-        files = __salt__['archive.cmd_unzip'](filename, name)
+        files = __salt__['archive.unzip'](filename, name)
     elif archive_format == 'rar':
         files = __salt__['archive.unrar'](filename, name)
     else:

salt/states/boto_route53.py

@@ -139,6 +139,11 @@ def present(
     '''
     ret = {'name': name, 'result': True, 'comment': '', 'changes': {}}
 
+    # If a list is passed in for value, change it to a comma-separated string
+    # So it will work with subsequent boto module calls and string functions
+    if isinstance(value, list):
+        value = ','.join(value)
+
     record = __salt__['boto_route53.get_record'](name, zone, record_type,
                                                  False, region, key, keyid,
                                                  profile, split_dns,

salt/states/file.py

@@ -1341,8 +1341,12 @@ def managed(name,
 
     if contents_pillar:
         contents = __salt__['pillar.get'](contents_pillar)
+        if not contents:
+            return _error(ret, 'contents_pillar {0} results in empty contents'.format(contents_pillar))
     if contents_grains:
         contents = __salt__['grains.get'](contents_grains)
+        if not contents:
+            return _error(ret, 'contents_grain {0} results in empty contents'.format(contents_grains))
 
     # ensure contents is a string
     if contents:
@@ -2481,7 +2485,9 @@ def replace(name,
         The replacement text.
 
     count
-        Maximum number of pattern occurrences to be replaced.
+        Maximum number of pattern occurrences to be replaced.  Defaults to 0.
+        If count is a positive integer n, no more than n occurrences will be
+        replaced, otherwise all occurrences will be replaced.
 
     flags
         A list of flags defined in the :ref:`re module documentation <contents-of-module-re>`.

salt/states/linux_acl.py

@@ -9,7 +9,7 @@ Ensure a Linux ACL is present
      root:
        acl.present:
          - name: /root
-         - acl_type: users
+         - acl_type: user
          - acl_name: damian
          - perms: rwx
 

tests/unit/modules/darwin_sysctl_test.py

@@ -95,7 +95,7 @@ class DarwinSysctlTestCase(TestCase):
         Tests successful write to existing sysctl file
         '''
         to_write = '#\n# Kernel sysctl configuration\n#\n'
-        m_calls_list = [call.writelines(['net.inet.icmp.icmplim=50'])]
+        m_calls_list = [call.writelines(['net.inet.icmp.icmplim=50', '\n'])]
         with patch('salt.utils.fopen', mock_open(read_data=to_write)) as m_open:
             darwin_sysctl.persist('net.inet.icmp.icmplim', 50, config=to_write)
             helper_open = m_open()

tests/unit/modules/defaults_test.py

@@ -30,16 +30,6 @@ class DefaultsTestCase(TestCase):
     '''
     Test cases for salt.modules.defaults
     '''
-    # 'get' function tests: 1
-
-    def test_get(self):
-        '''
-        Test if it execute a defaults client run and return a dict
-        '''
-        mock = MagicMock(return_value='')
-        with patch.dict(defaults.__salt__, {'pillar.get': mock}):
-            self.assertEqual(defaults.get('core:users:root'), '')
-
     @patch('salt.modules.defaults.get',
            MagicMock(return_value={'users': {'root': [0]}}))
     def test_get_mock(self):