Merge branch 'develop' into fix_broken_salt_return

salt/cloud/clouds/xen.py

@@ -7,6 +7,7 @@ XenServer Cloud Driver
 The XenServer driver is designed to work with a Citrix XenServer.
 
 Requires XenServer SDK
+(can be downloaded from https://www.citrix.com/downloads/xenserver/product-software/ )
 
 Place a copy of the XenAPI.py in the Python site-packages folder.
 
@@ -157,13 +158,27 @@ def _get_session():
         default=False,
         search_global=False
     )
-    session = XenAPI.Session(url, ignore_ssl=ignore_ssl)
-    log.debug('url: {} user: {} password: {}, originator: {}'.format(
-        url,
-        user,
-        'XXX-pw-redacted-XXX',
-        originator))
-    session.xenapi.login_with_password(user, password, api_version, originator)
+    try:
+        session = XenAPI.Session(url, ignore_ssl=ignore_ssl)
+        log.debug('url: {} user: {} password: {}, originator: {}'.format(
+            url,
+            user,
+            'XXX-pw-redacted-XXX',
+            originator))
+        session.xenapi.login_with_password(
+            user, password, api_version, originator)
+    except XenAPI.Failure as ex:
+        pool_master_addr = str(ex.__dict__['details'][1])
+        slash_parts = url.split('/')
+        new_url = '/'.join(slash_parts[:2]) + '/' + pool_master_addr
+        session = XenAPI.Session(new_url)
+        log.debug('session is -> url: {} user: {} password: {}, originator:{}'.format(
+            new_url,
+            user,
+            'XXX-pw-redacted-XXX',
+            originator))
+        session.xenapi.login_with_password(
+            user, password, api_version, originator)
     return session
 
 
@@ -182,14 +197,19 @@ def list_nodes():
     for vm in vms:
         record = session.xenapi.VM.get_record(vm)
         if not record['is_a_template'] and not record['is_control_domain']:
-            ret[record['name_label']] = {
-                'id': record['uuid'],
-                'image': record['other_config']['base_template_name'],
-                'name': record['name_label'],
-                'size': record['memory_dynamic_max'],
-                'state': record['power_state'],
-                'private_ips': get_vm_ip(record['name_label'], session),
-                'public_ips': None}
+            try:
+                base_template_name = record['other_config']['base_template_name']
+            except Exception:
+                base_template_name = None
+                log.debug('VM {}, doesnt have base_template_name attribute'.format(
+                    record['name_label']))
+            ret[record['name_label']] = {'id': record['uuid'],
+                                         'image': base_template_name,
+                                         'name': record['name_label'],
+                                         'size': record['memory_dynamic_max'],
+                                         'state': record['power_state'],
+                                         'private_ips': get_vm_ip(record['name_label'], session),
+                                         'public_ips': None}
     return ret
 
 
@@ -296,10 +316,17 @@ def list_nodes_full(session=None):
     for vm in vms:
         record = session.xenapi.VM.get_record(vm)
         if not record['is_a_template'] and not record['is_control_domain']:
+            # deal with cases where the VM doesn't have 'base_template_name' attribute
+            try:
+                base_template_name = record['other_config']['base_template_name']
+            except Exception:
+                base_template_name = None
+                log.debug('VM {}, doesnt have base_template_name attribute'.format(
+                    record['name_label']))
             vm_cfg = session.xenapi.VM.get_record(vm)
             vm_cfg['id'] = record['uuid']
             vm_cfg['name'] = record['name_label']
-            vm_cfg['image'] = record['other_config']['base_template_name']
+            vm_cfg['image'] = base_template_name
             vm_cfg['size'] = None
             vm_cfg['state'] = record['power_state']
             vm_cfg['private_ips'] = get_vm_ip(record['name_label'], session)
@@ -455,8 +482,14 @@ def show_instance(name, session=None, call=None):
     vm = _get_vm(name, session=session)
     record = session.xenapi.VM.get_record(vm)
     if not record['is_a_template'] and not record['is_control_domain']:
+        try:
+            base_template_name = record['other_config']['base_template_name']
+        except Exception:
+            base_template_name = None
+            log.debug('VM {}, doesnt have base_template_name attribute'.format(
+                record['name_label']))
         ret = {'id': record['uuid'],
-               'image': record['other_config']['base_template_name'],
+               'image': base_template_name,
                'name': record['name_label'],
                'size': record['memory_dynamic_max'],
                'state': record['power_state'],
@@ -716,7 +749,7 @@ def _copy_vm(template=None, name=None, session=None, sr=None):
     '''
     Create VM by copy
 
-    This is faster and should be used if source and target are
+    This is slower and should be used if source and target are
     NOT in the same storage repository
 
     template = object reference

salt/modules/boto_iam.py

@@ -42,6 +42,7 @@ from __future__ import absolute_import
 import logging
 import json
 import yaml
+import time
 
 # Import salt libs
 from salt.ext import six
@@ -2148,6 +2149,7 @@ def list_entities_for_policy(policy_name, path_prefix=None, entity_filter=None,
         salt myminion boto_iam.list_entities_for_policy mypolicy
     '''
     conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
+    retries = 30
 
     params = {}
     for arg in ('path_prefix', 'entity_filter'):
@@ -2155,21 +2157,26 @@ def list_entities_for_policy(policy_name, path_prefix=None, entity_filter=None,
             params[arg] = locals()[arg]
 
     policy_arn = _get_policy_arn(policy_name, region, key, keyid, profile)
-    try:
-        allret = {
-          'policy_groups': [],
-          'policy_users': [],
-          'policy_roles': [],
-        }
-        for ret in __utils__['boto.paged_call'](conn.list_entities_for_policy, policy_arn=policy_arn, **params):
-            for k, v in six.iteritems(allret):
-                v.extend(ret.get('list_entities_for_policy_response', {}).get('list_entities_for_policy_result', {}).get(k))
-        return allret
-    except boto.exception.BotoServerError as e:
-        log.debug(e)
-        msg = 'Failed to list {0} policy entities.'
-        log.error(msg.format(policy_name))
-        return {}
+    while retries:
+        try:
+            allret = {
+              'policy_groups': [],
+              'policy_users': [],
+              'policy_roles': [],
+            }
+            for ret in __utils__['boto.paged_call'](conn.list_entities_for_policy, policy_arn=policy_arn, **params):
+                for k, v in six.iteritems(allret):
+                    v.extend(ret.get('list_entities_for_policy_response', {}).get('list_entities_for_policy_result', {}).get(k))
+            return allret
+        except boto.exception.BotoServerError as e:
+            if e.error_code == 'Throttling':
+                log.debug("Throttled by AWS API, will retry in 5 seconds...")
+                time.sleep(5)
+                retries -= 1
+                continue
+            log.error('Failed to list {0} policy entities: {1}'.format(policy_name, e.message))
+            return {}
+    return {}
 
 
 def list_attached_user_policies(user_name, path_prefix=None, entity_filter=None,

salt/modules/kubernetes.py

@@ -83,7 +83,7 @@ def __virtual__():
     return False, 'python kubernetes library not found'
 
 
-if not salt.utils.is_windows():
+if not salt.utils.platform.is_windows():
     @contextmanager
     def _time_limit(seconds):
         def signal_handler(signum, frame):
@@ -713,7 +713,7 @@ def delete_deployment(name, namespace='default', **kwargs):
             namespace=namespace,
             body=body)
         mutable_api_response = api_response.to_dict()
-        if not salt.utils.is_windows():
+        if not salt.utils.platform.is_windows():
             try:
                 with _time_limit(POLLING_TIME_LIMIT):
                     while show_deployment(name, namespace) is not None:

salt/modules/purefa.py

@@ -30,7 +30,7 @@ Installation Prerequisites
 - Configure Pure Storage FlashArray authentication. Use one of the following
   three methods.
 
-    1) From the minion config
+  1) From the minion config
   .. code-block:: yaml
 
         pure_tags:
@@ -38,8 +38,8 @@ Installation Prerequisites
             san_ip: management vip or hostname for the FlashArray
             api_token: A valid api token for the FlashArray being managed
 
-    2) From environment (PUREFA_IP and PUREFA_API)
-    3) From the pillar (PUREFA_IP and PUREFA_API)
+  2) From environment (PUREFA_IP and PUREFA_API)
+  3) From the pillar (PUREFA_IP and PUREFA_API)
 
 :maintainer: Simon Dodsley (simon@purestorage.com)
 :maturity: new

salt/pillar/git_pillar.py

@@ -374,20 +374,20 @@ def __virtual__():
         return False
 
 
-def ext_pillar(minion_id, repo):
+def ext_pillar(minion_id, pillar, *repos):  # pylint: disable=unused-argument
     '''
     Checkout the ext_pillar sources and compile the resulting pillar SLS
     '''
     opts = copy.deepcopy(__opts__)
     opts['pillar_roots'] = {}
     opts['__git_pillar'] = True
-    pillar = salt.utils.gitfs.GitPillar(opts)
-    pillar.init_remotes(repo, PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY)
+    git_pillar = salt.utils.gitfs.GitPillar(opts)
+    git_pillar.init_remotes(repos, PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY)
     if __opts__.get('__role') == 'minion':
         # If masterless, fetch the remotes. We'll need to remove this once
         # we make the minion daemon able to run standalone.
-        pillar.fetch_remotes()
-    pillar.checkout()
+        git_pillar.fetch_remotes()
+    git_pillar.checkout()
     ret = {}
     merge_strategy = __opts__.get(
         'pillar_source_merging_strategy',
@@ -397,7 +397,7 @@ def ext_pillar(minion_id, repo):
         'pillar_merge_lists',
         False
     )
-    for pillar_dir, env in six.iteritems(pillar.pillar_dirs):
+    for pillar_dir, env in six.iteritems(git_pillar.pillar_dirs):
         # If pillarenv is set, only grab pillars with that match pillarenv
         if opts['pillarenv'] and env != opts['pillarenv']:
             log.debug(
@@ -406,7 +406,7 @@ def ext_pillar(minion_id, repo):
                 env, pillar_dir, opts['pillarenv']
             )
             continue
-        if pillar_dir in pillar.pillar_linked_dirs:
+        if pillar_dir in git_pillar.pillar_linked_dirs:
             log.debug(
                 'git_pillar is skipping processing on %s as it is a '
                 'mounted repo', pillar_dir
@@ -433,7 +433,7 @@ def ext_pillar(minion_id, repo):
             # list, so that its top file is sourced from the correct
             # location and not from another git_pillar remote.
             pillar_roots.extend(
-                [d for (d, e) in six.iteritems(pillar.pillar_dirs)
+                [d for (d, e) in six.iteritems(git_pillar.pillar_dirs)
                  if env == e and d != pillar_dir]
             )
 

tests/support/gitfs.py

@@ -341,7 +341,8 @@ class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin):
         with patch.dict(git_pillar.__opts__, ext_pillar_opts):
             return git_pillar.ext_pillar(
                 'minion',
-                ext_pillar_opts['ext_pillar'][0]['git'],
+                {},
+                *ext_pillar_opts['ext_pillar'][0]['git']
             )
 
     def make_repo(self, root_dir, user='root'):

tests/unit/modules/test_disk.py

@@ -152,6 +152,8 @@ class DiskTestCase(TestCase, LoaderModuleMockMixin):
         with patch.dict(disk.__salt__, {'cmd.retcode': mock}):
             self.assertEqual(disk.format_(device), True)
 
+    @skipIf(not salt.utils.which('lsblk') and not salt.utils.which('df'),
+            'lsblk or df not found')
     def test_fstype(self):
         '''
         unit tests for disk.fstype

tests/unit/modules/test_hosts.py

@@ -94,7 +94,7 @@ class HostsTestCase(TestCase, LoaderModuleMockMixin):
         Tests true if the alias is set
         '''
         hosts_file = '/etc/hosts'
-        if salt.utils.is_windows():
+        if salt.utils.platform.is_windows():
             hosts_file = r'C:\Windows\System32\Drivers\etc\hosts'
 
         with patch('salt.modules.hosts.__get_hosts_filename',
@@ -198,7 +198,7 @@ class HostsTestCase(TestCase, LoaderModuleMockMixin):
         Tests if specified host entry gets added from the hosts file
         '''
         hosts_file = '/etc/hosts'
-        if salt.utils.is_windows():
+        if salt.utils.platform.is_windows():
             hosts_file = r'C:\Windows\System32\Drivers\etc\hosts'
 
         with patch('salt.utils.files.fopen', mock_open()), \

tests/unit/returners/test_local_cache.py

@@ -97,7 +97,7 @@ class LocalCacheCleanOldJobsTestCase(TestCase, LoaderModuleMockMixin):
         local_cache.clean_old_jobs()
 
         # Get the name of the JID directory that was created to test against
-        if salt.utils.is_windows():
+        if salt.utils.platform.is_windows():
             jid_dir_name = jid_dir.rpartition('\\')[2]
         else:
             jid_dir_name = jid_dir.rpartition('/')[2]