Merge pull request #28709 from basepi/merge-forward-2015.8

[2015.8] Merge forward from 2015.5 to 2015.8

conf/minion

@@ -31,7 +31,9 @@
 # master_type: str
 
 # Poll interval in seconds for checking if the master is still there.  Only
-# respected if master_type above is "failover".
+# respected if master_type above is "failover". To disable the interval entirely,
+# set the value to -1. (This may be necessary on machines which have high numbers
+# of TCP connections, such as load balancers.)
 # master_alive_interval: 30
 
 # Set whether the minion should connect to the master via IPv6:

doc/ref/configuration/minion.rst

@@ -481,6 +481,35 @@ behavior is to have time-frame within all minions try to reconnect.
 
     recon_randomize: True
 
+.. conf_minion:: return_retry_timer
+
+``return_retry_timer``
+-------------------
+
+Default: ``5``
+
+The default timeout for a minion return attempt.
+
+.. code-block:: yaml
+
+    return_retry_timer: 5
+
+
+.. conf_minion:: return_retry_timer_max
+
+``return_retry_timer_max``
+-------------------
+
+Default: ``10``
+
+The maximum timeout for a minion return attempt. If non-zero the minion return
+retry timeout will be a random int beween ``return_retry_timer`` and
+``return_retry_timer_max``
+
+.. code-block:: yaml
+
+    return_retry_timer_max: 10
+
 .. conf_minion:: cache_sreqs
 
 ``cache_sreqs``

salt/config.py

@@ -371,7 +371,7 @@ VALID_OPTS = {
     'recon_randomize': float,  # FIXME This should really be a bool, according to the implementation
 
     'return_retry_timer': int,
-    'return_retry_random': bool,
+    'return_retry_timer_max': int,
 
     # Specify a returner in which all events will be sent to. Requires that the returner in question
     # have an event_return(event) function!
@@ -895,8 +895,8 @@ DEFAULT_MINION_OPTS = {
     'recon_max': 10000,
     'recon_default': 1000,
     'recon_randomize': True,
-    'return_retry_timer': 4,
-    'return_retry_random': True,
+    'return_retry_timer': 5,
+    'return_retry_timer_max': 10,
     'syndic_log_file': os.path.join(salt.syspaths.LOGS_DIR, 'syndic'),
     'syndic_pidfile': os.path.join(salt.syspaths.PIDFILE_DIR, 'salt-syndic.pid'),
     'random_reauth_delay': 10,

salt/minion.py

@@ -799,20 +799,22 @@ class Minion(MinionBase):
         just return the value of the return_retry_timer.
         '''
         msg = 'Minion return retry timer set to {0} seconds'
-        if self.opts.get('return_retry_random'):
+        if self.opts.get('return_retry_timer_max'):
             try:
-                random_retry = randint(1, self.opts['return_retry_timer'])
+                random_retry = randint(self.opts['return_retry_timer'], self.opts['return_retry_timer_max'])
+                log.debug(msg.format(random_retry) + ' (randomized)')
+                return random_retry
             except ValueError:
                 # Catch wiseguys using negative integers here
                 log.error(
-                    'Invalid value ({0}) for return_retry_timer, must be a '
-                    'positive integer'.format(self.opts['return_retry_timer'])
+                    'Invalid value (return_retry_timer: {0} or return_retry_timer_max: {1})'
+                    'both must be a positive integers'.format(
+                        self.opts['return_retry_timer'],
+                        self.opts['return_retry_timer_max'],
+                    )
                 )
                 log.debug(msg.format(DEFAULT_MINION_OPTS['return_retry_timer']))
                 return DEFAULT_MINION_OPTS['return_retry_timer']
-            else:
-                log.debug(msg.format(random_retry) + ' (randomized)')
-                return random_retry
         else:
             log.debug(msg.format(self.opts.get('return_retry_timer')))
             return self.opts.get('return_retry_timer')

salt/modules/file.py

@@ -1803,6 +1803,7 @@ def replace(path,
     if filesize is not 0:
         # First check the whole file, determine whether to make the replacement
         # Searching first avoids modifying the time stamp if there are no changes
+        r_data = None
         try:
             # Use a read-only handle to open the file
             with salt.utils.fopen(path,
@@ -1858,6 +1859,7 @@ def replace(path,
         except (OSError, IOError) as exc:
             raise CommandExecutionError("Exception: {0}".format(exc))
 
+        r_data = None
         try:
             # Open the file in write mode
             with salt.utils.fopen(path,

salt/modules/ps.py

@@ -100,7 +100,7 @@ def _get_proc_username(proc):
     '''
     try:
         return proc.username() if PSUTIL2 else proc.username
-    except (psutil.NoSuchProcess, psutil.AccessDenied):
+    except (psutil.NoSuchProcess, psutil.AccessDenied, KeyError):
         return None
 
 

salt/modules/rabbitmq.py

@@ -130,11 +130,9 @@ def list_vhosts(runas=None):
     '''
     if runas is None:
         runas = salt.utils.get_user()
-    res = __salt__['cmd.run']('rabbitmqctl list_vhosts',
-                              runas=runas)
-
-    # remove first and last line: Listing ... - ...done
-    return _strip_listing_to_done(res.splitlines())
+    res = __salt__['cmd.run']('rabbitmqctl list_vhosts -q',
+                              runas=runas).splitlines()
+    return res
 
 
 def user_exists(name, runas=None):

salt/modules/win_system.py

@@ -15,6 +15,8 @@ from datetime import datetime
 
 # Import 3rd Party Libs
 try:
+    import pythoncom
+    import wmi
     import win32net
     import win32api
     import win32con
@@ -412,14 +414,41 @@ def get_computer_desc():
 get_computer_description = salt.utils.alias_function(get_computer_desc, 'get_computer_description')
 
 
-def join_domain(
-        domain=None,
-        username=None,
-        password=None,
-        account_ou=None,
-        account_exists=False):
+def _lookup_error(number):
     '''
-    Join a computer to an Active Directory domain
+    Lookup the error based on the passed number
+    .. versionadded:: 2015.5.7
+    .. versionadded:: 2015.8.2
+
+    :param int number: Number code to lookup
+
+    :return: The text that corresponds to the error number
+    :rtype: str
+    '''
+    return_values = {
+        2:    'Invalid OU or specifying OU is not supported',
+        5:    'Access is denied',
+        53:   'The network path was not found',
+        87:   'The parameter is incorrect',
+        110:  'The system cannot open the specified object',
+        1323: 'Unable to update the password',
+        1326: 'Logon failure: unknown username or bad password',
+        1355: 'The specified domain either does not exist or could not be contacted',
+        2224: 'The account already exists',
+        2691: 'The machine is already joined to the domain',
+        2692: 'The machine is not currently joined to a domain',
+    }
+    return return_values[number]
+
+
+def join_domain(domain,
+                username=None,
+                password=None,
+                account_ou=None,
+                account_exists=False,
+                restart=False):
+    '''
+    Join a computer to an Active Directory domain. Requires reboot.
 
     :param str domain:
         The domain to which the computer should be joined, e.g.
@@ -441,6 +470,13 @@ def join_domain(
     :param bool account_exists:
         Needs to be set to ``True`` to allow re-using an existing account
 
+    :param bool restart: Restarts the computer after a successful join
+    .. versionadded:: 2015.5.7
+    .. versionadded:: 2015.8.2
+
+    :returns: Returns a dictionary if successful. False if unsuccessful.
+    :rtype: dict, bool
+
     CLI Example:
 
     .. code-block:: bash
@@ -448,98 +484,169 @@ def join_domain(
         salt 'minion-id' system.join_domain domain='domain.tld' \\
                          username='joinuser' password='joinpassword' \\
                          account_ou='ou=clients,ou=org,dc=domain,dc=tld' \\
-                         account_exists=False
+                         account_exists=False, restart=True
     '''
+    status = get_domain_workgroup()
+    if 'Domain' in status:
+        if status['Domain'] == domain:
+            return 'Already joined to {0}'.format(domain)
 
-    if '@' not in username:
+    if username and '\\' not in username and '@' not in username:
         username = '{0}@{1}'.format(username, domain)
 
+    if username and password is None:
+        return 'Must specify a password if you pass a username'
+
     # remove any escape characters
     if isinstance(account_ou, str):
         account_ou = account_ou.split('\\')
         account_ou = ''.join(account_ou)
 
-    join_options = 3
-    if account_exists:
-        join_options = 1
+    NETSETUP_JOIN_DOMAIN = 0x1
+    NETSETUP_ACCOUNT_CREATE = 0x2
+    NETSETUP_DOMAIN_JOIN_IF_JOINED = 0x20
 
-    ret = windll.netapi32.NetJoinDomain(None,
-                                        domain,
-                                        account_ou,
-                                        username,
-                                        password,
-                                        join_options)
-    if ret == 0:
-        return {'Domain': domain}
+    join_options = 0x0
+    join_options |= NETSETUP_JOIN_DOMAIN
+    join_options |= NETSETUP_DOMAIN_JOIN_IF_JOINED
+    if not account_exists:
+        join_options |= NETSETUP_ACCOUNT_CREATE
 
-    return_values = {
-        2:    'Invalid OU or specifying OU is not supported',
-        5:    'Access is denied',
-        53:   'The network path was not found',
-        87:   'The parameter is incorrect',
-        110:  'The system cannot open the specified object',
-        1323: 'Unable to update the password',
-        1326: 'Logon failure: unknown username or bad password',
-        1355: 'The specified domain either does not exist or could not be contacted',
-        2224: 'The account already exists',
-        2691: 'The machine is already joined to the domain',
-        2692: 'The machine is not currently joined to a domain',
-    }
-    log.error(return_values[ret])
+    pythoncom.CoInitialize()
+    c = wmi.WMI()
+    comp = c.Win32_ComputerSystem()[0]
+    err = comp.JoinDomainOrWorkgroup(Name=domain,
+                                     Password=password,
+                                     UserName=username,
+                                     AccountOU=account_ou,
+                                     FJoinOptions=join_options)
+
+    # you have to do this because JoinDomainOrWorkgroup returns a strangely
+    # formatted value that looks like (0,)
+    if not err[0]:
+        ret = {'Domain': domain,
+               'Restart': False}
+        if restart:
+            ret['Restart'] = reboot()
+        return ret
+
+    log.error(_lookup_error(err[0]))
     return False
 
 
-def unjoin_domain(username=None, password=None, disable=False):
-    '''
-    Unjoin a computer from an Active Directory Domain
+def unjoin_domain(username=None,
+                  password=None,
+                  domain=None,
+                  workgroup='WORKGROUP',
+                  disable=False,
+                  restart=False):
+    r'''
+    Unjoin a computer from an Active Directory Domain. Requires restart.
 
     :param username:
-        Username of an account which is authorized to join computers to the
-        specified domain. Need to be either fully qualified like
-        ``user@domain.tld`` or simply ``user``
+        Username of an account which is authorized to manage computer accounts
+        on the domain. Need to be fully qualified like ``user@domain.tld`` or
+        ``domain.tld\user``. If domain not specified, the passed domain will be
+        used. If computer account doesn't need to be disabled, can be None.
 
     :param str password:
         Password of the specified user
 
+    :param str domain: The domain from which to unjoin the computer. Can be None
+
+    :param str workgroup: The workgroup to join the computer to. Default is
+    ``WORKGROUP``
+
+    .. versionadded:: 2015.5.7
+    .. versionadded:: 2015.8.2
+
     :param bool disable:
         Disable the user account in Active Directory. True to disable.
 
-    :return: True if successful. False if not. Log contains error code.
-    :rtype: bool
+    :param bool restart: Restart the computer after successful unjoin
+
+    .. versionadded:: 2015.5.7
+    .. versionadded:: 2015.8.2
+
+    :returns: Returns a dictionary if successful. False if unsuccessful.
+    :rtype: dict, bool
 
     CLI Example:
 
     .. code-block:: bash
 
+        salt 'minion-id' system.unjoin_domain restart=True
+
         salt 'minion-id' system.unjoin_domain username='unjoinuser' \\
-                         password='unjoinpassword' disable=True
+                         password='unjoinpassword' disable=True \\
+                         restart=True
     '''
-    unjoin_options = 0
+    status = get_domain_workgroup()
+    if 'Workgroup' in status:
+        if status['Workgroup'] == workgroup:
+            return 'Already joined to {0}'.format(workgroup)
+
+    if username and '\\' not in username and '@' not in username:
+        if domain:
+            username = '{0}@{1}'.format(username, domain)
+        else:
+            return 'Must specify domain if not supplied in username'
+
+    if username and password is None:
+        return 'Must specify a password if you pass a username'
+
+    NETSETUP_ACCT_DELETE = 0x2
+
+    unjoin_options = 0x0
     if disable:
-        unjoin_options = 2
+        unjoin_options |= NETSETUP_ACCT_DELETE
 
-    ret = windll.netapi32.NetUnjoinDomain(None,
-                                          username,
-                                          password,
-                                          unjoin_options)
-    if ret == 0:
-        return True
+    pythoncom.CoInitialize()
+    c = wmi.WMI()
+    comp = c.Win32_ComputerSystem()[0]
+    err = comp.UnjoinDomainOrWorkgroup(Password=password,
+                                       UserName=username,
+                                       FUnjoinOptions=unjoin_options)
 
-    return_values = {
-        2:    'Invalid OU or specifying OU is not supported',
-        5:    'Access is denied',
-        53:   'The network path was not found',
-        87:   'The parameter is incorrect',
-        110:  'The system cannot open the specified object',
-        1323: 'Unable to update the password',
-        1326: 'Logon failure: unknown username or bad password',
-        1355: 'The specified domain either does not exist or could not be contacted',
-        2224: 'The account already exists',
-        2691: 'The machine is already joined to the domain',
-        2692: 'The machine is not currently joined to a domain',
-    }
-    log.error(return_values[ret])
-    return False
+    # you have to do this because UnjoinDomainOrWorkgroup returns a
+    # strangely formatted value that looks like (0,)
+    if not err[0]:
+        err = comp.JoinDomainOrWorkgroup(Name=workgroup)
+        if not err[0]:
+            ret = {'Workgroup': workgroup,
+                   'Restart': False}
+            if restart:
+                ret['Restart'] = reboot()
+
+            return ret
+        else:
+            log.error(_lookup_error(err[0]))
+            log.error('Failed to join the computer to {0}'.format(workgroup))
+            return False
+    else:
+        log.error(_lookup_error(err[0]))
+        log.error('Failed to unjoin computer from {0}'.format(status['Domain']))
+        return False
+
+
+def get_domain_workgroup():
+    '''
+    Get the domain or workgroup the computer belongs to.
+
+    .. versionadded:: 2015.5.7
+    .. versionadded:: 2015.8.2
+
+    :return: The name of the domain or workgroup
+    :rtype: str
+
+    '''
+    pythoncom.CoInitialize()
+    c = wmi.WMI()
+    for computer in c.Win32_ComputerSystem():
+        if computer.PartOfDomain:
+            return {'Domain': computer.Domain}
+        else:
+            return {'Workgroup': computer.Domain}
 
 
 def _get_date_time_format(dt_string):

salt/netapi/rest_tornado/saltnado.py

@@ -1612,7 +1612,12 @@ class WebhookSaltAPIHandler(SaltAPIHandler):  # pylint: disable=W0223
 
         ret = self.event.fire_event({
             'post': self.raw_data,
-            'headers': self.request.headers,
+            # In Tornado >= v4.0.3, the headers come
+            # back as an HTTPHeaders instance, which
+            # is a dictionary. We must cast this as
+            # a dictionary in order for msgpack to
+            # serialize it.
+            'headers': dict(self.request.headers),
         }, tag)
 
         self.write(self.serialize({'success': ret}))

salt/pillar/s3.py

@@ -181,16 +181,25 @@ def _init(creds, bucket, multiple_env, environment, prefix, s3_cache_expire):
     cache_file = _get_buckets_cache_filename(bucket, prefix)
     exp = time.time() - s3_cache_expire
 
-    # check mtime of the buckets files cache
-    cache_file_mtime = os.path.getmtime(cache_file)
-    if os.path.isfile(cache_file) and cache_file_mtime > exp:
-        log.debug("S3 bucket cache file {0} is not expired, mtime_diff={1}s, expiration={2}s".format(cache_file, cache_file_mtime - exp, s3_cache_expire))
-        return _read_buckets_cache_file(cache_file)
+    # check if cache_file exists and its mtime
+    if os.path.isfile(cache_file):
+        cache_file_mtime = os.path.getmtime(cache_file)
     else:
-        # bucket files cache expired
-        log.debug("S3 bucket cache file {0} is expired, mtime_diff={1}s, expiration={2}s".format(cache_file, cache_file_mtime - exp, s3_cache_expire))
-        return _refresh_buckets_cache_file(creds, cache_file, multiple_env,
+        # file does not exists then set mtime to 0 (aka epoch)
+        cache_file_mtime = 0
+
+    expired = (cache_file_mtime <= exp)
+
+    log.debug("S3 bucket cache file {0} is {1}expired, mtime_diff={2}s, expiration={3}s".format(cache_file, "" if expired else "not ", cache_file_mtime - exp, s3_cache_expire))
+
+    if expired:
+        pillars = _refresh_buckets_cache_file(creds, cache_file, multiple_env,
                                            environment, prefix)
+    else:
+        pillars = _read_buckets_cache_file(cache_file)
+
+    log.debug("S3 bucket retrieved pillars {0}".format(pillars))
+    return pillars
 
 
 def _get_cache_dir():

tests/unit/modules/rabbitmq_test.py

@@ -42,7 +42,7 @@ class RabbitmqTestCase(TestCase):
         with patch.dict(rabbitmq.__salt__, {'cmd.run': mock_run}):
             self.assertDictEqual(rabbitmq.list_users(), {'guest': set(['administrator'])})
 
-    # 'list_vhosts' function tests: 3
+    # 'list_vhosts' function tests: 1
 
     def test_list_vhosts(self):
         '''
@@ -52,22 +52,6 @@ class RabbitmqTestCase(TestCase):
         with patch.dict(rabbitmq.__salt__, {'cmd.run': mock_run}):
             self.assertListEqual(rabbitmq.list_vhosts(), ['...', 'saltstack', '...'])
 
-    def test_list_vhosts_trailing_done(self):
-        '''
-        Ensure any trailing '...done' line is stripped from output.
-        '''
-        mock_run = MagicMock(return_value='...\nsaltstack\n...done')
-        with patch.dict(rabbitmq.__salt__, {'cmd.run': mock_run}):
-            self.assertListEqual(rabbitmq.list_vhosts(), ['...', 'saltstack'])
-
-    def test_list_vhosts_succeeding_listing(self):
-        '''
-        Ensure succeeding 'Listing ...' line is stripped from output
-        '''
-        mock_run = MagicMock(return_value='Listing vhosts ...\nsaltstack\n...')
-        with patch.dict(rabbitmq.__salt__, {'cmd.run': mock_run}):
-            self.assertListEqual(rabbitmq.list_vhosts(), ['saltstack', '...'])
-
     # 'user_exists' function tests: 2
 
     def test_user_exists_negative(self):