saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #33945 from rallytime/merge-2016.3

Browse source 
[2016.3] Merge forward from 2015.8 to 2016.3
This commit is contained in:
Nicole Thomas 2016-06-13 11:44:30 -04:00 • committed by GitHub
commit 81e16bb93f
10 changed files with 233 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
salt/cloud/__init__.py
View file

@ -1285,29 +1285,34 @@ class Cloud(object):
log.error('Bad option for sync_after_install')
return output
# a small pause makes the sync work reliably
# A small pause helps the sync work more reliably
time.sleep(3)
mopts_ = salt.config.DEFAULT_MINION_OPTS
conf_path = '/'.join(self.opts['conf_file'].split('/')[:-1])
mopts_.update(
salt.config.minion_config(
os.path.join(conf_path,
'minion')
start = int(time.time())
while int(time.time()) < start + 60:
# We'll try every <timeout> seconds, up to a minute
mopts_ = salt.config.DEFAULT_MINION_OPTS
conf_path = '/'.join(self.opts['conf_file'].split('/')[:-1])
mopts_.update(
salt.config.minion_config(
os.path.join(conf_path,
'minion')
)
)
)
client = salt.client.get_local_client(mopts=self.opts)
client = salt.client.get_local_client(mopts=self.opts)
ret = client.cmd(
vm_['name'],
'saltutil.sync_{0}'.format(self.opts['sync_after_install']),
timeout=self.opts['timeout']
)
log.info(
six.u('Synchronized the following dynamic modules: '
' {0}').format(ret)
)
ret = client.cmd(
vm_['name'],
'saltutil.sync_{0}'.format(self.opts['sync_after_install']),
timeout=self.opts['timeout']
)
if ret:
log.info(
six.u('Synchronized the following dynamic modules: '
' {0}').format(ret)
)
break
except KeyError as exc:
log.exception(
'Failed to create VM {0}. Configuration value {1} needs '

2
salt/modules/random_org.py
View file

@ -120,7 +120,7 @@ def _query(api_version=None, data=None):
elif result.get('status', None) == salt.ext.six.moves.http_client.NO_CONTENT:
return False
else:
ret['message'] = result.text
ret['message'] = result.text if hasattr(result, 'text') else ''
return ret

66
salt/modules/saltutil.py
View file

@ -187,11 +187,12 @@ def sync_beacons(saltenv=None, refresh=True):
will be performed even if no new beacons are synced. Set to ``False``
to prevent this refresh.
CLI Examples:
CLI Example:
.. code-block:: bash
salt '*' saltutil.sync_beacons
salt '*' saltutil.sync_beacons saltenv=dev
salt '*' saltutil.sync_beacons saltenv=base,dev
'''
ret = _sync('beacons', saltenv)
@ -210,11 +211,16 @@ def sync_sdb(saltenv=None):
The fileserver environment from which to sync. To sync from more than
one environment, pass a comma-separated list.
CLI Examples:
refresh : False
This argument has no affect and is included for consistency with the
other sync functions.
CLI Example:
.. code-block:: bash
salt '*' saltutil.sync_sdb
salt '*' saltutil.sync_sdb saltenv=dev
salt '*' saltutil.sync_sdb saltenv=base,dev
'''
ret = _sync('sdb', saltenv)
@ -253,11 +259,12 @@ def sync_modules(saltenv=None, refresh=True):
See :ref:`here <reloading-modules>` for a more detailed explanation of
why this is necessary.
CLI Examples:
CLI Example:
.. code-block:: bash
salt '*' saltutil.sync_modules
salt '*' saltutil.sync_modules saltenv=dev
salt '*' saltutil.sync_modules saltenv=base,dev
'''
ret = _sync('modules', saltenv)
@ -286,6 +293,7 @@ def sync_states(saltenv=None, refresh=True):
.. code-block:: bash
salt '*' saltutil.sync_states
salt '*' saltutil.sync_states saltenv=dev
salt '*' saltutil.sync_states saltenv=base,dev
'''
ret = _sync('states', saltenv)
@ -315,6 +323,7 @@ def sync_grains(saltenv=None, refresh=True):
.. code-block:: bash
salt '*' saltutil.sync_grains
salt '*' saltutil.sync_grains saltenv=dev
salt '*' saltutil.sync_grains saltenv=base,dev
'''
ret = _sync('grains', saltenv)
@ -345,6 +354,7 @@ def sync_renderers(saltenv=None, refresh=True):
.. code-block:: bash
salt '*' saltutil.sync_renderers
salt '*' saltutil.sync_renderers saltenv=dev
salt '*' saltutil.sync_renderers saltenv=base,dev
'''
ret = _sync('renderers', saltenv)
@ -373,6 +383,7 @@ def sync_returners(saltenv=None, refresh=True):
.. code-block:: bash
salt '*' saltutil.sync_returners
salt '*' saltutil.sync_returners saltenv=dev
'''
ret = _sync('returners', saltenv)
if refresh:
@ -400,6 +411,7 @@ def sync_proxymodules(saltenv=None, refresh=False):
.. code-block:: bash
salt '*' saltutil.sync_proxymodules
salt '*' saltutil.sync_proxymodules saltenv=dev
salt '*' saltutil.sync_proxymodules saltenv=base,dev
'''
ret = _sync('proxy', saltenv)
@ -454,6 +466,7 @@ def sync_output(saltenv=None, refresh=True):
.. code-block:: bash
salt '*' saltutil.sync_output
salt '*' saltutil.sync_output saltenv=dev
salt '*' saltutil.sync_output saltenv=base,dev
'''
ret = _sync('output', saltenv)
@ -484,6 +497,7 @@ def sync_utils(saltenv=None, refresh=True):
.. code-block:: bash
salt '*' saltutil.sync_utils
salt '*' saltutil.sync_utils saltenv=dev
salt '*' saltutil.sync_utils saltenv=base,dev
'''
ret = _sync('utils', saltenv)
@ -512,6 +526,7 @@ def sync_log_handlers(saltenv=None, refresh=True):
.. code-block:: bash
salt '*' saltutil.sync_log_handlers
salt '*' saltutil.sync_log_handlers saltenv=dev
salt '*' saltutil.sync_log_handlers saltenv=base,dev
'''
ret = _sync('log_handlers', saltenv)
@ -520,8 +535,47 @@ def sync_log_handlers(saltenv=None, refresh=True):
return ret
def sync_pillar(saltenv=None, refresh=True):
'''
.. versionadded:: 2015.8.11,2016.3.2
Sync pillar modules from the ``salt://_pillar`` directory on the Salt
fileserver. This function is environment-aware, pass the desired
environment to grab the contents of the ``_pillar`` directory from that
environment. The default environment, if none is specified, is ``base``.
refresh : True
Also refresh the execution modules available to the minion, and refresh
pillar data.
.. note::
This function will raise an error if executed on a traditional (i.e.
not masterless) minion
CLI Examples:
.. code-block:: bash
salt '*' saltutil.sync_pillar
salt '*' saltutil.sync_pillar saltenv=dev
'''
if __opts__['file_client'] != 'local':
raise CommandExecutionError(
'Pillar modules can only be synced to masterless minions'
)
ret = _sync('pillar', saltenv)
if refresh:
refresh_modules()
refresh_pillar()
return ret
def sync_all(saltenv=None, refresh=True):
'''
.. versionchanged:: 2015.8.11,2016.3.2
On masterless minions, pillar modules are now synced, and refreshed
when ``refresh`` is set to ``True``.
Sync down all of the dynamic modules from the file server for a specific
environment. This function synchronizes custom modules, states, beacons,
grains, returners, output modules, renderers, and utils.
@ -553,6 +607,7 @@ def sync_all(saltenv=None, refresh=True):
.. code-block:: bash
salt '*' saltutil.sync_all
salt '*' saltutil.sync_all saltenv=dev
salt '*' saltutil.sync_all saltenv=base,dev
'''
log.debug('Syncing all')
@ -569,9 +624,12 @@ def sync_all(saltenv=None, refresh=True):
ret['log_handlers'] = sync_log_handlers(saltenv, False)
ret['proxymodules'] = sync_proxymodules(saltenv, False)
ret['engines'] = sync_engines(saltenv, False)
if __opts__['file_client'] == 'local':
ret['pillar'] = sync_pillar(saltenv, False)
if refresh:
refresh_modules()
refresh_pillar()
if __opts__['file_client'] == 'local':
refresh_pillar()
return ret

2
salt/returners/sqlite3_return.py
View file

@ -212,7 +212,7 @@ def get_load(jid):
{'jid': jid})
data = cur.fetchone()
if data:
return json.loads(data)
return json.loads(data[0].encode())
_close_conn(conn)
return {}

142
salt/states/archive.py
View file

@ -15,8 +15,10 @@ from contextlib import closing
# Import 3rd-party libs
import salt.ext.six as six
from salt.ext.six.moves import shlex_quote as _cmd_quote
# Import salt libs
import salt.utils
from salt.exceptions import CommandExecutionError
import salt.utils
# remove after archive_user deprecation.
@ -66,6 +68,21 @@ def compareChecksum(fname, target, checksum):
return False
def _is_bsdtar():
return 'bsdtar' in __salt__['cmd.run'](['tar', '--version'],
python_shell=False)
def _cleanup_destdir(name):
'''
Attempt to remove the specified directory
'''
try:
os.rmdir(name)
except OSError:
pass
def extracted(name,
source,
archive_format,
@ -136,7 +153,7 @@ def extracted(name,
- if_missing: /opt/graylog2-server-0.9.6p1/
name
Directory name where to extract the archive
Location where archive should be extracted
password
Password to use with password protected zip files. Currently only zip
@ -183,25 +200,41 @@ def extracted(name,
be enforced if ``if_missing`` is a directory.
if_missing
Some archives, such as tar, extract themselves in a subfolder.
This directive can be used to validate if the archive had been
previously extracted.
If specified, this path will be checked, and if it exists then the
archive will not be extracted. This can be helpful if the archive
extracts all files into a subfolder. This path can be either a
directory or a file, so this option can also be used to check for a
semaphore file and conditionally skip extraction.
.. versionchanged:: 2016.3.0
When used in combination with either ``user`` or ``group``,
ownership will only be enforced when ``if_missing`` is a directory.
tar_options
Required if used with ``archive_format: tar``, otherwise optional.
It needs to be the tar argument specific to the archive being extracted,
such as 'J' for LZMA or 'v' to verbosely list files processed.
Using this option means that the tar executable on the target will
be used, which is less platform independent.
Main operators like -x, --extract, --get, -c and -f/--file
**should not be used** here.
If ``archive_format`` is ``zip`` or ``rar`` and this option is not set,
then the Python tarfile module is used. The tarfile module supports gzip
and bz2 in Python 2.
If ``archive_format`` is set to ``tar``, this option can be used to
specify a string of additional arguments to pass to the tar command. If
``archive_format`` is set to ``tar`` and this option is *not* used,
then the minion will attempt to use Python's native tarfile_ support to
extract it. Python's native tarfile_ support can only handle gzip and
bzip2 compression, however.
.. versionchanged:: 2015.8.11,2016.3.2
XZ-compressed archives no longer require ``J`` to manually be set
in the ``tar_options``, they are now detected automatically and
Salt will extract them using ``xz-utils``. This is a more
platform-independent solution, as not all tar implementations
support the ``J`` argument for extracting archives.
.. note::
Main operators like -x, --extract, --get, -c and -f/--file **should
not be used** here.
Using this option means that the ``tar`` command will be used,
which is less platform-independent, so keep this in mind when using
this option; the options must be valid options for the ``tar``
implementation on the minion's OS.
.. _tarfile: https://docs.python.org/2/library/tarfile.html
keep
Keep the archive in the minion's cache
@ -308,7 +341,7 @@ def extracted(name,
log.debug('failed to download {0}'.format(source))
return file_result
else:
log.debug('Archive %s is already in cache', name)
log.debug('Archive %s is already in cache', source)
if __opts__['test']:
ret['result'] = None
@ -320,7 +353,15 @@ def extracted(name,
)
return ret
__salt__['file.makedirs'](name, user=user, group=group)
created_destdir = False
if __salt__['file.file_exists'](name.rstrip('/')):
ret['result'] = False
ret['comment'] = ('{0} exists and is not a directory'
.format(name.rstrip('/')))
return ret
elif not __salt__['file.directory_exists'](name):
__salt__['file.makedirs'](name, user=archive_user)
created_destdir = True
log.debug('Extracting {0} to {1}'.format(filename, name))
if archive_format == 'zip':
@ -329,11 +370,70 @@ def extracted(name,
files = __salt__['archive.unrar'](filename, name, trim_output=trim_output)
else:
if tar_options is None:
with closing(tarfile.open(filename, 'r')) as tar:
files = tar.getnames()
tar.extractall(name)
try:
with closing(tarfile.open(filename, 'r')) as tar:
files = tar.getnames()
tar.extractall(name)
except tarfile.ReadError:
if salt.utils.which('xz'):
if __salt__['cmd.retcode'](['xz', '-l', filename],
python_shell=False,
ignore_retcode=True) == 0:
# XZ-compressed data
log.debug(
'Tar file is XZ-compressed, attempting '
'decompression and extraction using xz-utils '
'and the tar command'
)
# Must use python_shell=True here because not all tar
# implementations support the -J flag for decompressing
# XZ-compressed data. We need to dump the decompressed
# data to stdout and pipe it to tar for extraction.
cmd = 'xz --decompress --stdout {0} | tar xvf -'
results = __salt__['cmd.run_all'](
cmd.format(_cmd_quote(filename)),
cwd=name,
python_shell=True)
if results['retcode'] != 0:
if created_destdir:
_cleanup_destdir(name)
ret['result'] = False
ret['changes'] = results
return ret
if _is_bsdtar():
files = results['stderr']
else:
files = results['stdout']
else:
# Failed to open tar archive and it is not
# XZ-compressed, gracefully fail the state
if created_destdir:
_cleanup_destdir(name)
ret['result'] = False
ret['comment'] = (
'Failed to read from tar archive using Python\'s '
'native tar file support. If archive is '
'compressed using something other than gzip or '
'bzip2, the \'tar_options\' parameter may be '
'required to pass the correct options to the tar '
'command in order to extract the archive.'
)
return ret
else:
if created_destdir:
_cleanup_destdir(name)
ret['result'] = False
ret['comment'] = (
'Failed to read from tar archive. If it is '
'XZ-compressed, install xz-utils to attempt '
'extraction.'
)
return ret
else:
tar_opts = tar_options.split(' ')
try:
tar_opts = tar_options.split(' ')
except AttributeError:
tar_opts = str(tar_options).split(' ')
tar_cmd = ['tar']
tar_shortopts = 'x'
@ -359,7 +459,7 @@ def extracted(name,
ret['result'] = False
ret['changes'] = results
return ret
if 'bsdtar' in __salt__['cmd.run']('tar --version', python_shell=False):
if _is_bsdtar():
files = results['stderr']
else:
files = results['stdout']

2
salt/states/boto_elb.py
View file

@ -71,6 +71,8 @@ passed in as a dict, or as a string to pull from pillars or minion config:
s3_bucket_name: 'mybucket'
s3_bucket_prefix: 'my-logs'
emit_interval: 5
connecting_settings:
idle_timeout: 60
- cnames:
- name: mycname.example.com.
zone: example.com.

5
salt/utils/http.py
View file

@ -507,6 +507,11 @@ def query(url,
ret['status'] = exc.code
ret['error'] = str(exc)
return ret
except socket.gaierror as exc:
if status is True:
ret['status'] = 0
ret['error'] = str(exc)
return ret
if stream is True or handle is True:
return {

9
salt/utils/master.py
View file

@ -690,8 +690,13 @@ class ConnectedCache(MultiprocessingProcess):
def ping_all_connected_minions(opts):
client = salt.client.LocalClient()
ckminions = salt.utils.minions.CkMinions(opts)
client.cmd(list(ckminions.connected_ids()), 'test.ping', expr_form='list')
if opts['minion_data_cache']:
tgt = list(salt.utils.minions.CkMinions(opts).connected_ids())
form = 'list'
else:
tgt = '*'
form = 'glob'
client.cmd(tgt, 'test.ping', expr_form=form)
# test code for the ConCache class
if __name__ == '__main__':

1
tests/integration/modules/sysmod.py
View file

@ -67,6 +67,7 @@ class SysModuleTest(integration.ModuleCase):
'runtests_decorators.missing_depends',
'runtests_decorators.missing_depends_will_fallback',
'swift.head',
'glance.warn_until',
'yumpkg.expand_repo_def',
'yumpkg5.expand_repo_def',
'container_resource.run',

10
tests/unit/modules/random_org_test.py
View file

@ -18,11 +18,21 @@ from salttesting.helpers import ensure_in_syspath
ensure_in_syspath('../../')
# Import Salt Libs
import salt.utils.http
from salt.modules import random_org
random_org.__opts__ = {}
def check_status():
'''
Check the status of random.org
'''
ret = salt.utils.http.query('https://api.random.org/', status=True)
return ret['status'] == 200
@skipIf(not check_status(), 'random.org is not available')
@skipIf(NO_MOCK, NO_MOCK_REASON)
class RandomOrgTestCase(TestCase):
'''