saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge branch '2018.3' into merge-fluorine

Browse source 
Conflicts:
	doc/conf.py
	doc/topics/development/dunder_dictionaries.rst
	salt/client/ssh/ssh_py_shim.py
	salt/grains/core.py
	salt/modules/mysql.py
	salt/modules/state.py
This commit is contained in:
Gareth J. Greenaway 2018-12-20 17:20:42 -08:00
commit 9c04986641
No known key found for this signature in database
GPG key ID: 10B62F8A7CAD7A41
26 changed files with 953 additions and 526 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

150
doc/conf.py
View file

@ -3,7 +3,6 @@
'''
Sphinx documentation for Salt
'''
import functools
import sys
import os
import re
@ -13,7 +12,6 @@ import time
from sphinx.directives import TocTree
# pylint: disable=R0903
class Mock(object):
'''
Mock out specified imports.
@ -25,7 +23,7 @@ class Mock(object):
http://read-the-docs.readthedocs.org/en/latest/faq.html#i-get-import-errors-on-libraries-that-depend-on-c-modules
'''
def __init__(self, mapping=None, *args, **kwargs):
def __init__(self, mapping=None, *args, **kwargs): # pylint: disable=unused-argument
"""
Mapping allows autodoc to bypass the Mock object, but actually assign
a specific value, expected by a specific attribute returned.
@ -41,14 +39,12 @@ class Mock(object):
return Mock(mapping=self.__mapping)
def __getattr__(self, name):
#__mapping = {'total': 0}
data = None
if name in self.__mapping:
data = self.__mapping.get(name)
elif name in ('__file__', '__path__'):
data = '/dev/null'
elif name == '__qualname__':
raise AttributeError("'Mock' object has no attribute '__qualname__'")
elif name in ('__mro_entries__', '__qualname__'):
raise AttributeError("'Mock' object has no attribute '%s'" % (name))
else:
data = Mock(mapping=self.__mapping)
return data
@ -56,10 +52,30 @@ class Mock(object):
def __iter__(self):
return self
def next(self):
@staticmethod
def __next__():
raise StopIteration
# pylint: enable=R0903
# For Python 2
next = __next__
def mock_decorator_with_params(*oargs, **okwargs): # pylint: disable=unused-argument
'''
Optionally mock a decorator that takes parameters
E.g.:
@blah(stuff=True)
def things():
pass
'''
def inner(fn, *iargs, **ikwargs): # pylint: disable=unused-argument
if hasattr(fn, '__call__'):
return fn
return Mock()
return inner
MOCK_MODULES = [
# Python stdlib
@ -147,7 +163,6 @@ MOCK_MODULES = [
'jnpr.junos.utils',
'jnpr.junos.utils.config',
'jnpr.junos.utils.sw',
'json',
'keyring',
'libvirt',
'lxml',
@ -182,42 +197,24 @@ MOCK_MODULES = [
'zfs',
]
MOCK_MODULES_MAPPING = {
'cherrypy': {'config': mock_decorator_with_params},
'ntsecuritycon': {
'STANDARD_RIGHTS_REQUIRED': 0,
'SYNCHRONIZE': 0,
},
'psutil': {'total': 0}, # Otherwise it will crash Sphinx
}
for mod_name in MOCK_MODULES:
if mod_name == 'psutil':
mock = Mock(mapping={'total': 0}) # Otherwise it will crash Sphinx
else:
mock = Mock()
sys.modules[mod_name] = mock
def mock_decorator_with_params(*oargs, **okwargs):
'''
Optionally mock a decorator that takes parameters
E.g.:
@blah(stuff=True)
def things():
pass
'''
def inner(fn, *iargs, **ikwargs):
if hasattr(fn, '__call__'):
return fn
else:
return Mock()
return inner
sys.modules[mod_name] = Mock(mapping=MOCK_MODULES_MAPPING.get(mod_name))
# Define a fake version attribute for the following libs.
sys.modules['libcloud'].__version__ = '0.0.0'
sys.modules['msgpack'].version = (1, 0, 0)
sys.modules['psutil'].version_info = (3, 0, 0)
sys.modules['pymongo'].version = '0.0.0'
sys.modules['ntsecuritycon'].STANDARD_RIGHTS_REQUIRED = 0
sys.modules['ntsecuritycon'].SYNCHRONIZE = 0
# Define a fake version attribute for the following libs.
sys.modules['cherrypy'].config = mock_decorator_with_params
sys.modules['tornado'].version_info = (0, 0, 0)
sys.modules['boto.regioninfo']._load_json_file = {'endpoints': None}
# -- Add paths to PYTHONPATH ---------------------------------------------------
@ -229,12 +226,12 @@ except NameError:
docs_basepath = os.path.abspath(os.path.dirname('.'))
addtl_paths = (
os.pardir, # salt itself (for autodoc)
'_ext', # custom Sphinx extensions
os.pardir, # salt itself (for autodoc)
'_ext', # custom Sphinx extensions
)
for path in addtl_paths:
sys.path.insert(0, os.path.abspath(os.path.join(docs_basepath, path)))
for addtl_path in addtl_paths:
sys.path.insert(0, os.path.abspath(os.path.join(docs_basepath, addtl_path)))
# We're now able to import salt
@ -314,7 +311,7 @@ extensions = [
]
try:
import sphinxcontrib.spelling
import sphinxcontrib.spelling # false positive, pylint: disable=unused-import
except ImportError:
pass
else:
@ -381,7 +378,8 @@ gettext_compact = False
### HTML options
html_theme = os.environ.get('HTML_THEME', 'saltstack2') # set 'HTML_THEME=saltstack' to use previous theme
# set 'HTML_THEME=saltstack' to use previous theme
html_theme = os.environ.get('HTML_THEME', 'saltstack2')
html_theme_path = ['_themes']
html_title = u''
html_short_title = 'Salt'
@ -449,7 +447,7 @@ html_show_copyright = True
### Latex options
latex_documents = [
('contents', 'Salt.tex', 'Salt Documentation', 'SaltStack, Inc.', 'manual'),
('contents', 'Salt.tex', 'Salt Documentation', 'SaltStack, Inc.', 'manual'),
]
latex_logo = '_static/salt-logo.png'
@ -457,7 +455,7 @@ latex_logo = '_static/salt-logo.png'
latex_elements = {
'inputenc': '', # use XeTeX instead of the inputenc LaTeX package.
'utf8extra': '',
'preamble': '''
'preamble': r'''
\usepackage{fontspec}
\setsansfont{Linux Biolinum O}
\setromanfont{Linux Libertine O}
@ -469,35 +467,36 @@ latex_elements = {
### Linkcheck options
linkcheck_ignore = [r'http://127.0.0.1',
r'http://salt:\d+',
r'http://local:\d+',
r'https://console.aws.amazon.com',
r'http://192.168.33.10',
r'http://domain:\d+',
r'http://123.456.789.012:\d+',
r'http://localhost',
r'https://groups.google.com/forum/#!forum/salt-users',
r'http://logstash.net/docs/latest/inputs/udp',
r'http://logstash.net/docs/latest/inputs/zeromq',
r'http://www.youtube.com/saltstack',
r'https://raven.readthedocs.io',
r'https://getsentry.com',
r'https://salt-cloud.readthedocs.io',
r'https://salt.readthedocs.io',
r'http://www.pip-installer.org/',
r'http://www.windowsazure.com/',
r'https://github.com/watching',
r'dash-feed://',
r'https://github.com/saltstack/salt/',
r'http://bootstrap.saltstack.org',
r'https://bootstrap.saltstack.com',
r'https://raw.githubusercontent.com/saltstack/salt-bootstrap/stable/bootstrap-salt.sh',
r'media.readthedocs.org/dash/salt/latest/salt.xml',
r'https://portal.aws.amazon.com/gp/aws/securityCredentials',
r'https://help.github.com/articles/fork-a-repo',
r'dash-feed://https%3A//media.readthedocs.org/dash/salt/latest/salt.xml'
]
linkcheck_ignore = [
r'http://127.0.0.1',
r'http://salt:\d+',
r'http://local:\d+',
r'https://console.aws.amazon.com',
r'http://192.168.33.10',
r'http://domain:\d+',
r'http://123.456.789.012:\d+',
r'http://localhost',
r'https://groups.google.com/forum/#!forum/salt-users',
r'http://logstash.net/docs/latest/inputs/udp',
r'http://logstash.net/docs/latest/inputs/zeromq',
r'http://www.youtube.com/saltstack',
r'https://raven.readthedocs.io',
r'https://getsentry.com',
r'https://salt-cloud.readthedocs.io',
r'https://salt.readthedocs.io',
r'http://www.pip-installer.org/',
r'http://www.windowsazure.com/',
r'https://github.com/watching',
r'dash-feed://',
r'https://github.com/saltstack/salt/',
r'http://bootstrap.saltstack.org',
r'https://bootstrap.saltstack.com',
r'https://raw.githubusercontent.com/saltstack/salt-bootstrap/stable/bootstrap-salt.sh',
r'media.readthedocs.org/dash/salt/latest/salt.xml',
r'https://portal.aws.amazon.com/gp/aws/securityCredentials',
r'https://help.github.com/articles/fork-a-repo',
r'dash-feed://https%3A//media.readthedocs.org/dash/salt/latest/salt.xml',
]
linkcheck_anchors = False
@ -541,6 +540,7 @@ epub_tocdup = False
def skip_mod_init_member(app, what, name, obj, skip, options):
# pylint: disable=too-many-arguments,unused-argument
if name.startswith('_'):
return True
if isinstance(obj, types.FunctionType) and obj.__name__ == 'mod_init':

57
doc/ref/file_server/dynamic-modules.rst
View file

@ -1,57 +0,0 @@
.. _module-sync:
.. _dynamic-module-distribution:
===========================
Dynamic Module Distribution
===========================
.. versionadded:: 0.9.5
Custom Salt execution, state, and other modules can be distributed to Salt
minions using the Salt file server.
Under the root of any environment defined via the :conf_master:`file_roots`
option on the master server directories corresponding to the type of module can
be used.
The directories are prepended with an underscore:
- :file:`_beacons`
- :file:`_clouds`
- :file:`_engines`
- :file:`_grains`
- :file:`_modules`
- :file:`_output`
- :file:`_proxy`
- :file:`_renderers`
- :file:`_returners`
- :file:`_states`
- :file:`_tops`
- :file:`_utils`
The contents of these directories need to be synced over to the minions after
Python modules have been created in them. There are a number of ways to sync
the modules.
Sync Via States
===============
The minion configuration contains an option ``autoload_dynamic_modules``
which defaults to ``True``. This option makes the state system refresh all
dynamic modules when states are run. To disable this behavior set
:conf_minion:`autoload_dynamic_modules` to ``False`` in the minion config.
When dynamic modules are autoloaded via states, modules only pertinent to
the environments matched in the master's top file are downloaded.
This is important to remember, because modules can be manually loaded from
any specific environment that environment specific modules will be loaded
when a state run is executed.
Sync Via the saltutil Module
============================
The saltutil module has a number of functions that can be used to sync all
or specific dynamic modules. The saltutil module function ``saltutil.sync_all``
will sync all module types over to a minion. For more information see:
:mod:`salt.modules.saltutil`

6
doc/topics/beacons/index.rst
View file

@ -290,6 +290,8 @@ All beacons are configured using a similar process of enabling the beacon,
writing a reactor SLS (and state SLS if needed), and mapping a beacon event to
the reactor SLS.
.. _writing-beacons:
Writing Beacon Plugins
======================
@ -360,5 +362,5 @@ new execution modules and functions to back specific beacons.
Distributing Custom Beacons
---------------------------
Custom beacons can be distributed to minions using ``saltutil``, see
:ref:`Dynamic Module Distribution <dynamic-module-distribution>`.
Custom beacons can be distributed to minions via the standard methods, see
:ref:`Modular Systems <modular-systems>`.

124
doc/topics/development/dunder_dictionaries.rst
View file

@ -1,124 +0,0 @@
.. _dunder-dictionaries:
===================
Dunder Dictionaries
===================
Salt provides several special "dunder" dictionaries as a convenience for Salt
development. These include ``__opts__``, ``__context__``, ``__salt__``, and
others. This document will describe each dictionary and detail where they exist
and what information and/or functionality they provide.
__opts__
--------
Available in
~~~~~~~~~~~~
- All loader modules
The ``__opts__`` dictionary contains all of the options passed in the
configuration file for the master or minion.
.. note::
In many places in salt, instead of pulling raw data from the __opts__
dict, configuration data should be pulled from the salt `get` functions
such as config.get, aka - __salt__['config.get']('foo:bar')
The `get` functions also allow for dict traversal via the *:* delimiter.
Consider using get functions whenever using __opts__ or __pillar__ and
__grains__ (when using grains for configuration data)
The configuration file data made available in the ``__opts__`` dictionary is the
configuration data relative to the running daemon. If the modules are loaded and
executed by the master, then the master configuration data is available, if the
modules are executed by the minion, then the minion configuration is
available. Any additional information passed into the respective configuration
files is made available
__salt__
--------
Available in
~~~~~~~~~~~~
- Execution Modules
- State Modules
- Returners
- Runners
- SDB Modules
``__salt__`` contains the execution module functions. This allows for all
functions to be called as they have been set up by the salt loader.
.. code-block:: python
__salt__['cmd.run']('fdisk -l')
__salt__['network.ip_addrs']()
.. note::
When used in runners, ``__salt__`` references other runner modules, and not
execution modules.
__grains__
----------
Available in
~~~~~~~~~~~~
- Execution Modules
- State Modules
- Returners
- External Pillar
The ``__grains__`` dictionary contains the grains data generated by the minion
that is currently being worked with. In execution modules, state modules and
returners this is the grains of the minion running the calls, when generating
the external pillar the ``__grains__`` is the grains data from the minion that
the pillar is being generated for.
__pillar__
-----------
Available in
~~~~~~~~~~~~
- Execution Modules
- State Modules
- Returners
The ``__pillar__`` dictionary contains the pillar for the respective minion.
.. _dunder-context:
__context__
-----------
``__context__`` exists in state modules and execution modules.
During a state run the ``__context__`` dictionary persists across all states
that are run and then is destroyed when the state ends.
When running an execution module ``__context__`` persists across all module
executions until the modules are refreshed; such as when
:py:func:`saltutil.sync_all <salt.modules.saltutil.sync_all>` or
:py:func:`state.apply <salt.modules.state.apply_>` are executed.
A great place to see how to use ``__context__`` is in the cp.py module in
salt/modules/cp.py. The fileclient authenticates with the master when it is
instantiated and then is used to copy files to the minion. Rather than create a
new fileclient for each file that is to be copied down, one instance of the
fileclient is instantiated in the ``__context__`` dictionary and is reused for
each file. Here is an example from salt/modules/cp.py:
.. code-block:: python
if not 'cp.fileclient' in __context__:
__context__['cp.fileclient'] = salt.fileclient.get_file_client(__opts__)
.. note:: Because __context__ may or may not have been destroyed, always be
sure to check for the existence of the key in __context__ and
generate the key before using it.

1
doc/topics/development/index.rst
View file

@ -7,6 +7,7 @@ Developing Salt
:glob:
*
modules/index
extend/index
tests/*
raet/index

170
doc/topics/development/modular_systems.rst
View file

@ -1,170 +0,0 @@
===============
Modular Systems
===============
When first working with Salt, it is not always clear where all of the modular
components are and what they do. Salt comes loaded with more modular systems
than many users are aware of, making Salt very easy to extend in many places.
The most commonly used modular systems are execution modules and states. But
the modular systems extend well beyond the more easily exposed components
and are often added to Salt to make the complete system more flexible.
Execution Modules
=================
Execution modules make up the core of the functionality used by Salt to
interact with client systems. The execution modules create the core system
management library used by all Salt systems, including states, which
interact with minion systems.
Execution modules are completely open ended in their execution. They can
be used to do anything required on a minion, from installing packages to
detecting information about the system. The only restraint in execution
modules is that the defined functions always return a JSON serializable
object.
For a list of all built in execution modules, click :ref:`here
<all-salt.modules>`
For information on writing execution modules, see :ref:`this page
<writing-execution-modules>`.
Interactive Debugging
=====================
Sometimes debugging with ``print()`` and extra logs sprinkled everywhere is not
the best strategy.
IPython is a helpful debug tool that has an interactive python environment
which can be embedded in python programs.
First the system will require IPython to be installed.
.. code-block:: bash
# Debian
apt-get install ipython
# Arch Linux
pacman -Syu ipython2
# RHEL/CentOS (via EPEL)
yum install python-ipython
Now, in the troubling python module, add the following line at a location where
the debugger should be started:
.. code-block:: python
test = 'test123'
import IPython; IPython.embed_kernel()
After running a Salt command that hits that line, the following will show up in
the log file:
.. code-block:: text
[CRITICAL] To connect another client to this kernel, use:
[IPKernelApp] --existing kernel-31271.json
Now on the system that invoked ``embed_kernel``, run the following command from
a shell:
.. code-block:: bash
# NOTE: use ipython2 instead of ipython for Arch Linux
ipython console --existing
This provides a console that has access to all the vars and functions, and even
supports tab-completion.
.. code-block:: python
print(test)
test123
To exit IPython and continue running Salt, press ``Ctrl-d`` to logout.
State Modules
=============
State modules are used to define the state interfaces used by Salt States.
These modules are restrictive in that they must follow a number of rules to
function properly.
.. note::
State modules define the available routines in sls files. If calling
an execution module directly is desired, take a look at the `module`
state.
Auth
====
The auth module system allows for external authentication routines to be easily
added into Salt. The `auth` function needs to be implemented to satisfy the
requirements of an auth module. Use the ``pam`` module as an example.
Fileserver
==========
The fileserver module system is used to create fileserver backends used by the
Salt Master. These modules need to implement the functions used in the
fileserver subsystem. Use the ``gitfs`` module as an example.
Grains
======
Grain modules define extra routines to populate grains data. All defined
public functions will be executed and MUST return a Python dict object. The
dict keys will be added to the grains made available to the minion.
Output
======
The output modules supply the outputter system with routines to display data
in the terminal. These modules are very simple and only require the `output`
function to execute. The default system outputter is the ``nested`` module.
Pillar
======
Used to define optional external pillar systems. The pillar generated via
the filesystem pillar is passed into external pillars. This is commonly used
as a bridge to database data for pillar, but is also the backend to the libvirt
state used to generate and sign libvirt certificates on the fly.
Renderers
=========
Renderers are the system used to render sls files into salt highdata for the
state compiler. They can be as simple as the ``py`` renderer and as complex as
``stateconf`` and ``pydsl``.
Returners
=========
Returners are used to send data from minions to external sources, commonly
databases. A full returner will implement all routines to be supported as an
external job cache. Use the ``redis`` returner as an example.
Runners
=======
Runners are purely master-side execution sequences.
Tops
====
Tops modules are used to convert external data sources into top file data for
the state system.
Wheel
=====
The wheel system is used to manage master side management routines. These
routines are primarily intended for the API to enable master configuration.

25
doc/topics/development/modules/configuration.rst Normal file
View file

@ -0,0 +1,25 @@
=====================
Configuration Options
=====================
A number of configuration options can affect the load process. This is a quick
list of them:
* ``autoload_dynamic_modules`` (:conf_minion:`Minion <autoload_dynamic_modules>`)
* ``cython_enable`` (:conf_minion:`Minion <cython_enable>`, :conf_master:`Master <cython_enable>`)
* ``disable_modules`` (:conf_minion:`Minion <disable_modules>`)
* ``disable_returners`` (:conf_minion:`Minion <disable_returners>`)
* ``enable_zip_modules`` (:conf_minion:`Minion <enable_zip_modules>`)
* ``extension_modules`` (:conf_master:`Master <extension_modules>`)
* ``extmod_whitelist`` (:conf_minion:`Minion <extmod_whitelist>`, :conf_master:`Master <extmod_whitelist>`)
* ``extmod_blacklist`` (:conf_minion:`Minion <extmod_blacklist>`, :conf_master:`Master <extmod_blacklist>`)
* ``whitelist_modules`` (:conf_minion:`Minion <enable_whitelist_modules>`)
* ``grains_dirs`` (:conf_minion:`Minion <grains_dirs>`)
* ``module_dirs`` (:conf_minion:`Minion <module_dirs>`, :conf_master:`Master <module_dirs>`)
* ``outputter_dirs`` (:conf_minion:`Minion <outputter_dirs>`, :conf_master:`Master <outputter_dirs>`)
* ``providers`` (:conf_minion:`Minion <providers>`)
* ``render_dirs`` (:conf_minion:`Minion <render_dirs>`)
* ``returner_dirs`` (:conf_minion:`Minion <returner_dirs>`)
* ``runner_dirs`` (:conf_master:`Master <runner_dirs>`)
* ``states_dirs`` (:conf_minion:`Minion <states_dirs>`)
* ``utils_dirs`` (:conf_minion:`Minion <utils_dirs>`)

237
doc/topics/development/modules/developing.rst Normal file
View file

@ -0,0 +1,237 @@
======================
Developing New Modules
======================
Interactive Debugging
=====================
Sometimes debugging with ``print()`` and extra logs sprinkled everywhere is not
the best strategy.
IPython is a helpful debug tool that has an interactive python environment
which can be embedded in python programs.
First the system will require IPython to be installed.
.. code-block:: bash
# Debian
apt-get install ipython
# Arch Linux
pacman -Syu ipython2
# RHEL/CentOS (via EPEL)
yum install python-ipython
Now, in the troubling python module, add the following line at a location where
the debugger should be started:
.. code-block:: python
test = 'test123'
import IPython; IPython.embed_kernel()
After running a Salt command that hits that line, the following will show up in
the log file:
.. code-block:: text
[CRITICAL] To connect another client to this kernel, use:
[IPKernelApp] --existing kernel-31271.json
Now on the system that invoked ``embed_kernel``, run the following command from
a shell:
.. code-block:: bash
# NOTE: use ipython2 instead of ipython for Arch Linux
ipython console --existing
This provides a console that has access to all the vars and functions, and even
supports tab-completion.
.. code-block:: python
print(test)
test123
To exit IPython and continue running Salt, press ``Ctrl-d`` to logout.
Special Module Contents
=======================
These are things that may be defined by the module to influence various things.
__virtual__
-----------
__virtual_aliases__
-------------------
__virtualname__
---------------
__init__
--------
Called before ``__virtual__()``
__proxyenabled__
----------------
grains and proxy modules
__proxyenabled__ as a list containing the names of the proxy types that the module supports.
__load__
--------
__func_alias__
--------------
__outputter__
-------------
.. _dunder-dictionaries:
Dunder Dictionaries
===================
Salt provides several special "dunder" dictionaries as a convenience for Salt
development. These include ``__opts__``, ``__context__``, ``__salt__``, and
others. This document will describe each dictionary and detail where they exist
and what information and/or functionality they provide.
The following dunder dictionaries are always defined, but may be empty
* ``__context__``
* ``__grains__``
* ``__pillar__``
* ``__opts__``
__opts__
--------
Defined in: All modules
The ``__opts__`` dictionary contains all of the options passed in the
configuration file for the master or minion.
.. note::
In many places in salt, instead of pulling raw data from the __opts__
dict, configuration data should be pulled from the salt `get` functions
such as config.get, aka - ``__salt__['config.get']('foo:bar')``
The `get` functions also allow for dict traversal via the *:* delimiter.
Consider using get functions whenever using ``__opts__`` or ``__pillar__``
and ``__grains__`` (when using grains for configuration data)
The configuration file data made available in the ``__opts__`` dictionary is the
configuration data relative to the running daemon. If the modules are loaded and
executed by the master, then the master configuration data is available, if the
modules are executed by the minion, then the minion configuration is
available. Any additional information passed into the respective configuration
files is made available
__salt__
--------
Defined in: Auth, Beacons, Engines, Execution, Executors, Outputters, Pillars,
Proxies, Renderers, Returners, Runners, SDB, SSH Wrappers, State, Thorium
``__salt__`` contains the execution module functions. This allows for all
functions to be called as they have been set up by the salt loader.
.. code-block:: python
__salt__['cmd.run']('fdisk -l')
__salt__['network.ip_addrs']()
.. note::
When used in runners or outputters, ``__salt__`` references other
runner/outputter modules, and not execution modules.
__grains__
----------
Filled in for: Execution, Pillar, Renderer, Returner, SSH Wrapper, State.
The ``__grains__`` dictionary contains the grains data generated by the minion
that is currently being worked with. In execution modules, state modules and
returners this is the grains of the minion running the calls, when generating
the external pillar the ``__grains__`` is the grains data from the minion that
the pillar is being generated for.
While ``__grains__`` is defined for every module, it's only filled in for some.
__pillar__
-----------
Filled in for: Execution, Returner, SSH Wrapper, State
The ``__pillar__`` dictionary contains the pillar for the respective minion.
While ``__pillar__`` is defined for every module, it's only filled in for some.
__context__
-----------
During a state run the ``__context__`` dictionary persists across all states
that are run and then is destroyed when the state ends.
When running an execution module ``__context__`` persists across all module
executions until the modules are refreshed; such as when
:py:func:`saltutil.sync_all <salt.modules.saltutil.sync_all>` or
:py:func:`state.apply <salt.modules.state.apply_>` are executed.
A great place to see how to use ``__context__`` is in the cp.py module in
salt/modules/cp.py. The fileclient authenticates with the master when it is
instantiated and then is used to copy files to the minion. Rather than create a
new fileclient for each file that is to be copied down, one instance of the
fileclient is instantiated in the ``__context__`` dictionary and is reused for
each file. Here is an example from salt/modules/cp.py:
.. code-block:: python
if not 'cp.fileclient' in __context__:
__context__['cp.fileclient'] = salt.fileclient.get_file_client(__opts__)
.. note:: Because __context__ may or may not have been destroyed, always be
sure to check for the existence of the key in __context__ and
generate the key before using it.
__utils__
---------
Defined in: Cloud, Engine, Execution, File Server, Pillar, Proxy, Runner, SDB.
__proxy__
---------
Defined in: Beacon, Engine, Execution, Executor, Proxy, Renderer, Returner, State, Util
__runners__
-----------
Defined in: Engine, Roster, Thorium
__ret__
-------
Defined in: Proxy, Search
__thorium__
-----------
Defined in: Thorium
__states__
----------
Defined in: Renderers, State
__serializers__
---------------
Defined in: State
__sdb__
-------
Defined in: SDB

2
doc/topics/development/external_pillars.rst → doc/topics/development/modules/external_pillars.rst
View file

@ -153,7 +153,7 @@ Using our example above:
ext_pillar( id, pillar, 'some argument' ) # example_a
ext_pillar( id, pillar, 'argumentA', 'argumentB' ) # example_b
ext_pillar( id, pillar, keyA='valueA', keyB='valueB' } ) # example_c
ext_pillar( id, pillar, keyA='valueA', keyB='valueB' ) # example_c
In the ``example_a`` case, ``pillar`` will contain the items from the

394
doc/topics/development/modules/index.rst Normal file
View file

@ -0,0 +1,394 @@
.. _modular-systems:
===============
Modular Systems
===============
When first working with Salt, it is not always clear where all of the modular
components are and what they do. Salt comes loaded with more modular systems
than many users are aware of, making Salt very easy to extend in many places.
The most commonly used modular systems are execution modules and states. But
the modular systems extend well beyond the more easily exposed components
and are often added to Salt to make the complete system more flexible.
.. toctree::
:maxdepth: 2
:glob:
developing
configuration
Loading Modules
===============
Modules come primarily from several sources:
* The Salt package itself
* The Salt File Server
* The extmods directory
* Secondary packages installed
Using one source to override another is not supported.
The Salt Package
----------------
Salt itself ships with a large number of modules. These are part of the Salt
package itself and don't require the user to do anything to use them. (Although
a number of them have additional dependencies and/or configuration.)
The Salt File Server
--------------------
The user may add modules by simply placing them in special directories in their
:ref:`fileserver <file-server>`.
The name of the directory inside of the file server is the directory name
prepended by underscore, such as:
- :file:`_grains`
- :file:`_modules`
- :file:`_states`
Modules must be synced before they can be used. This can happen a few ways,
discussed below.
.. note:
Using saltenvs besides ``base`` may not work in all contexts.
Sync Via States
~~~~~~~~~~~~~~~
The minion configuration contains an option ``autoload_dynamic_modules``
which defaults to ``True``. This option makes the state system refresh all
dynamic modules when states are run. To disable this behavior set
:conf_minion:`autoload_dynamic_modules` to ``False`` in the minion config.
When dynamic modules are autoloaded via states, only the modules defined in the
same saltenvs as the states currently being run.
Sync Via the saltutil Module
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The saltutil module has a number of functions that can be used to sync all
or specific dynamic modules. The ``saltutil.sync_*``
:py:mod:`execution functions <salt.modules.saltutil>` and
:py:mod:`runner functions <salt.runners.saltutil>` can be used to sync modules
to minions and the master, respectively.
The extmods Directory
---------------------
Any files places in the directory set by ``extension_modules`` settings
(:conf_minion:`minion <extension_modules>`,
:conf_master:`master <extension_modules>`, default
``/var/cache/salt/*/extmods``) can also be loaded as modules. Note that these
directories are also used by the ``saltutil.sync_*`` functions (mentioned
above) and files may be overwritten.
Secondary Packages
------------------
Third-party packages may also add modules to Salt if they are installed in the
same system and Python environment as the Salt Minion or Master.
This is done via setuptools entry points:
.. code-block:: python
setup(
# ...
entry_points={
'salt.loader': [
'module_dirs=spirofs.loader:module',
],
},
# ...
)
Note that these are not synced from the Salt Master to the Minions. They must be
installed indepdendently on each Minion.
Module Types
============
The specific names used by each loading method above are as follows. See sections below
for a short summary of each of these systems.
.. _module-name-table:
============ ================================================================ ========================= =====================
Module Type Salt Package Name FS/Directory Name Entry Point
============ ================================================================ ========================= =====================
Auth ``salt.auth`` (:ref:`index <external-logging-handlers>`) ``auth`` [#no-fs]_ ``auth_dirs``
Beacon ``salt.beacons`` (:ref:`index <beacons>`) ``beacons`` ``beacons_dirs``
Cache ``salt.cache`` (:ref:`index <all-salt.cache>`) ``cache`` ``cache_dirs``
Cloud ``salt.cloud.clouds`` (:ref:`index <all-salt.clouds>`) ``clouds`` ``cloud_dirs``
Engine ``salt.engines`` (:ref:`index <engines>`) ``engines`` ``engines_dirs``
Execution ``salt.modules`` (:ref:`index <all-salt.modules>`) ``modules`` ``module_dirs``
Executor ``salt.executors`` (:ref:`index <all-salt.executors>`) ``executors`` [#no-fs]_ ``executor_dirs``
File Server ``salt.fileserver`` (:ref:`index <file-server>`) ``fileserver`` [#no-fs]_ ``fileserver_dirs``
Grain ``salt.grains`` (:ref:`index <all-salt.grains>`) ``grains`` ``grains_dirs``
Log Handler ``salt.log.handlers`` (:ref:`index <external-logging-handlers>`) ``log_handlers`` ``log_handlers_dirs``
Net API ``salt.netapi`` (:ref:`index <all-netapi-modules>`) ``netapi`` [#no-fs]_ ``netapi_dirs``
Outputter ``salt.output`` (:ref:`index <all-salt.output>`) ``output`` ``outputter_dirs``
Pillar ``salt.pillar`` (:ref:`index <all-salt.pillars>`) ``pillar`` ``pillar_dirs``
Proxy ``salt.proxy`` (:ref:`index <all-salt.proxy>`) ``proxy`` ``proxy_dirs``
Queue ``salt.queues`` (:ref:`index <all-salt.queues>`) ``queues`` ``queue_dirs``
Renderer ``salt.renderers`` (:ref:`index <all-salt.renderers>`) ``renderers`` ``render_dirs``
Returner ``salt.returners`` (:ref:`index <all-salt.returners>`) ``returners`` ``returner_dirs``
Roster ``salt.roster`` (:ref:`index <all-salt.roster>`) ``roster`` ``roster_dirs``
Runner ``salt.runners`` (:ref:`index <all-salt.runners>`) ``runners`` ``runner_dirs``
SDB ``salt.sdb`` (:ref:`index <all-salt.sdb>`) ``sdb`` ``sdb_dirs``
Search ``salt.search`` ``search`` [#no-fs]_ ``search_dirs``
Serializer ``salt.serializers`` (:ref:`index <all-salt.serializers>`) ``serializers`` [#no-fs]_ ``serializers_dirs``
SPM pkgdb ``salt.spm.pkgdb`` ``pkgdb`` [#no-fs]_ ``pkgdb_dirs``
SPM pkgfiles ``salt.spm.pkgfiles`` ``pkgfiles`` [#no-fs]_ ``pkgfiles_dirs``
SSH Wrapper ``salt.client.ssh.wrapper`` ``wrapper`` [#no-fs]_ ``wrapper_dirs``
State ``salt.states`` (:ref:`index <all-salt.states>`) ``states`` ``states_dirs``
Thorium ``salt.thorium`` (:ref:`index <all-salt.thorium>`) ``thorium`` [#no-fs]_ ``thorium_dirs``
Top ``salt.tops`` (:ref:`index <all-salt.tops>`) ``tops`` ``top_dirs``
Util ``salt.utils`` ``utils`` ``utils_dirs``
Wheel ``salt.wheels`` (:ref:`index <all-salt.wheel>`) ``wheel`` ``wheel_dirs``
============ ================================================================ ========================= =====================
.. [#no-fs] These modules cannot be loaded from the Salt File Server.
.. note:
While it is possible to import modules directly with the import statement,
it is strongly recommended that the appropriate
:ref:`dunder dictionary <dunder-dictionaries>` is used to access them
instead. This is because a number of factors affect module names, module
selection, and module overloading.
Auth
----
The auth module system allows for external authentication routines to be easily
added into Salt. The `auth` function needs to be implemented to satisfy the
requirements of an auth module. Use the ``pam`` module as an example.
See :ref:`External Authentication System <acl-eauth>` for more about
authentication in Salt.
Beacon
------
* :ref:`Writing Beacons <writing-beacons>`
Beacons are polled by the Salt event loop to monitor non-salt processes. See
:ref:`Beacons <beacons>` for more information about the beacon system.
Cache
-----
The minion cache is used by the master to store various information about
minions. See :ref:`Minion Data Cache <cache>` for more information.
Cloud
-----
Cloud modules are backend implementations used by :ref:`Salt Cloud <salt-cloud>`.
Engine
------
Engines are open-ended services managed by the Salt daemon (both master and
minion). They may interact with event loop, call other modules, or a variety of
non-salt tasks. See :ref:`Salt Engines <engines>` for complete details.
Execution
---------
.. toctree::
:maxdepth: 1
:glob:
/ref/modules/index
Execution modules make up the core of the functionality used by Salt to
interact with client systems. The execution modules create the core system
management library used by all Salt systems, including states, which
interact with minion systems.
Execution modules are completely open ended in their execution. They can
be used to do anything required on a minion, from installing packages to
detecting information about the system. The only restraint in execution
modules is that the defined functions always return a JSON serializable
object.
Executor
--------
Executors control how execution modules get called. The default is to just call
them, but this can be customized.
File Server
-----------
The file server module system is used to create file server backends used by the
Salt Master. These modules need to implement the functions used in the
fileserver subsystem. Use the ``gitfs`` module as an example.
See :ref:`File Server Backends <file-server-backends>` for more information.
Grains
------
* :ref:`writing-grains`
Grain modules define extra routines to populate grains data. All defined
public functions will be executed and MUST return a Python dict object. The
dict keys will be added to the grains made available to the minion.
See :ref:`Grains <grains>` for more.
Log Handler
-----------
Log handlers allows the logs from salt (master or minion) to be sent to log
aggregation systems.
Net API
-------
Net API modules are the actual server implementation used by Salt API.
Output
------
The output modules supply the outputter system with routines to display data
in the terminal. These modules are very simple and only require the `output`
function to execute. The default system outputter is the ``nested`` module.
Pillar
------
.. toctree::
:maxdepth: 1
:glob:
external_pillars
Used to define optional external pillar systems. The pillar generated via
the filesystem pillar is passed into external pillars. This is commonly used
as a bridge to database data for pillar, but is also the backend to the libvirt
state used to generate and sign libvirt certificates on the fly.
Proxy
-----
:ref:`Proxy Minions <proxy-minion>` are a way to manage devices that cannot run
a full minion directly.
Renderers
---------
Renderers are the system used to render sls files into salt highdata for the
state compiler. They can be as simple as the ``py`` renderer and as complex as
``stateconf`` and ``pydsl``.
Returners
---------
Returners are used to send data from minions to external sources, commonly
databases. A full returner will implement all routines to be supported as an
external job cache. Use the ``redis`` returner as an example.
Roster
------
The :ref:`Roster system <ssh-roster>` is used by Salt SSH to enumerate devices.
Runners
-------
.. toctree::
:maxdepth: 1
:glob:
/ref/runners/index
Runners are purely master-side execution sequences.
SDB
---
* :ref:`Writing SDB Modules <sdb-writing-modules>`
SDB is a way to store data that's not associated with a minion. See
:ref:`Storing Data in Other Databases <sdb>`.
Search
------
A system for indexing the file server and pillars. Removed in 2018.3.
Serializer
----------
Primarily used with :py:func:`file.serialize <salt.states.file.serialize>`.
State
-----
.. toctree::
:maxdepth: 1
:glob:
/ref/states/index
State modules are used to define the state interfaces used by Salt States.
These modules are restrictive in that they must follow a number of rules to
function properly.
.. note::
State modules define the available routines in sls files. If calling
an execution module directly is desired, take a look at the `module`
state.
SPM pkgdb
---------
* :ref:`SPM Development Guide: Package Database <spm-development-pkgdb>`
pkgdb modules provides storage backends to the package database.
SPM pkgfiles
------------
* :ref:`SPM Development Guide: Package Database <spm-development-pkgfiles>`
pkgfiles modules handle the actual installation.
SSH Wrapper
-----------
Replacement execution modules for :ref:`Salt SSH <salt-ssh>`.
Thorium
-------
Modules for use in the :ref:`Thorium <thorium-reactor>` event reactor.
Tops
----
Tops modules are used to convert external data sources into top file data for
the state system.
Util
----
Just utility modules to use with other modules via ``__utils__`` (see
:ref:`Dunder Dictionaries <dunder-dictionaries>`).
Wheel
-----
The wheel system is used to manage master side management routines. These
routines are primarily intended for the API to enable master configuration.

1
doc/topics/sdb/index.rst
View file

@ -144,6 +144,7 @@ When writing Salt modules, it is not recommended to call ``sdb.get`` directly,
as it requires the user to provide values in SDB, using a specific URI. Use
``config.get`` instead.
.. _sdb-writing-modules:
Writing SDB Modules
===================

3
doc/topics/spm/dev.rst
View file

@ -20,6 +20,7 @@ marked as required are crucial to SPM's core functionality, while arguments that
are marked as optional are provided as a benefit to the module, if it needs to
use them.
.. _spm-development-pkgdb:
Package Database
----------------
@ -146,6 +147,8 @@ The only argument that is expected is ``db_``, which is the package database
file.
.. _spm-development-pkgfiles:
Package Files
-------------
By default, package files are installed using the ``local`` module. This module

6
doc/topics/targeting/index.rst
View file

@ -90,9 +90,9 @@ shorthand for having to type out complicated compound expressions.
.. code-block:: yaml
nodegroups:
 group1: 'L@foo.domain.com,bar.domain.com,baz.domain.com and bl*.domain.com'
 group2: 'G@os:Debian and foo.domain.com'
 group3: 'G@os:Debian and N@group1'
group1: 'L@foo.domain.com,bar.domain.com,baz.domain.com and bl*.domain.com'
group2: 'G@os:Debian and foo.domain.com'
group3: 'G@os:Debian and N@group1'
Advanced Targeting Methods

15
salt/client/ssh/ssh_py_shim.py
View file

@ -339,22 +339,23 @@ def main(argv): # pylint: disable=W0613
if OPTIONS.cmd_umask is not None:
old_umask = os.umask(OPTIONS.cmd_umask) # pylint: disable=blacklisted-function
if OPTIONS.tty:
proc = subprocess.Popen(salt_argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# Returns bytes instead of string on python 3
stdout, _ = subprocess.Popen(salt_argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
if sys.version_info[0] == 2 and sys.version_info[1] < 7:
sys.stdout.write(stdout.decode(get_system_encoding(), "replace"))
else:
sys.stdout.write(stdout.decode(encoding=get_system_encoding(), errors="replace"))
stdout, _ = proc.communicate()
sys.stdout.write(stdout.decode(encoding=get_system_encoding(), errors="replace"))
sys.stdout.flush()
retcode = proc.returncode
if OPTIONS.wipe:
shutil.rmtree(OPTIONS.saltdir)
elif OPTIONS.wipe:
subprocess.call(salt_argv)
retcode = subprocess.call(salt_argv)
shutil.rmtree(OPTIONS.saltdir)
else:
subprocess.call(salt_argv)
retcode = subprocess.call(salt_argv)
if OPTIONS.cmd_umask is not None:
os.umask(old_umask) # pylint: disable=blacklisted-function
return retcode
if __name__ == '__main__':

42
salt/grains/core.py
View file

@ -1234,26 +1234,34 @@ def _windows_platform_data():
os_release = platform.release()
kernel_version = platform.version()
info = salt.utils.win_osinfo.get_os_version_info()
net_info = salt.utils.win_osinfo.get_join_info()
server = {'Vista': '2008Server',
'7': '2008ServerR2',
'8': '2012Server',
'8.1': '2012ServerR2',
'10': '2016Server'}
# Starting with Python 2.7.12 and 3.5.2 the `platform.uname()` function
# started reporting the Desktop version instead of the Server version on
# Server versions of Windows, so we need to look those up
# So, if you find a Server Platform that's a key in the server
# dictionary, then lookup the actual Server Release.
# (Product Type 1 is Desktop, Everything else is Server)
if info['ProductType'] > 1 and os_release in server:
os_release = server[os_release]
service_pack = None
if info['ServicePackMajor'] > 0:
service_pack = ''.join(['SP', six.text_type(info['ServicePackMajor'])])
# This creates the osrelease grain based on the Windows Operating
# System Product Name. As long as Microsoft maintains a similar format
# this should be future proof
version = 'Unknown'
release = ''
if 'Server' in osinfo.Caption:
for item in osinfo.Caption.split(' '):
# If it's all digits, then it's version
if re.match(r'\d+', item):
version = item
# If it starts with R and then numbers, it's the release
# ie: R2
if re.match(r'^R\d+$', item):
release = item
os_release = '{0}Server{1}'.format(version, release)
else:
for item in osinfo.Caption.split(' '):
# If it's a number, decimal number, Thin or Vista, then it's the
# version
if re.match(r'^(\d+(\.\d+)?)|Thin|Vista$', item):
version = item
os_release = version
grains = {
'kernelrelease': _clean_value('kernelrelease', osinfo.Version),
'kernelversion': _clean_value('kernelversion', kernel_version),
@ -1269,8 +1277,8 @@ def _windows_platform_data():
'serialnumber': _clean_value('serialnumber', biosinfo.SerialNumber),
'osfullname': _clean_value('osfullname', osinfo.Caption),
'timezone': _clean_value('timezone', timeinfo.Description),
'windowsdomain': _clean_value('windowsdomain', net_info['Domain']),
'windowsdomaintype': _clean_value('windowsdomaintype', net_info['DomainType']),
'windowsdomain': _clean_value('windowsdomain', systeminfo.Domain),
'windowsdomaintype': _clean_value('windowsdomaintype', systeminfo.DomainType),
'motherboard': {
'productname': _clean_value('motherboard.productname', motherboard['product']),
'serialnumber': _clean_value('motherboard.serialnumber', motherboard['serial']),

2
salt/modules/file.py
View file

@ -793,6 +793,7 @@ def get_source_sum(file_name='',
ret = extract_hash(hash_fn, '', file_name, source, source_hash_name)
if ret is None:
_invalid_source_hash_format()
ret['hsum'] = ret['hsum'].lower()
return ret
else:
# The source_hash is a hash expression
@ -836,6 +837,7 @@ def get_source_sum(file_name='',
)
)
ret['hsum'] = ret['hsum'].lower()
return ret

38
salt/modules/mysql.py
View file

@ -330,7 +330,7 @@ def _connect(**kwargs):
try:
dbc = MySQLdb.connect(**connargs)
except OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return None
@ -646,7 +646,7 @@ def query(database, query, **connection_args):
try:
affected = _execute(cur, query)
except OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return False
@ -771,7 +771,7 @@ def status(**connection_args):
try:
_execute(cur, qry)
except OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return {}
@ -802,7 +802,7 @@ def version(**connection_args):
try:
_execute(cur, qry)
except MySQLdb.OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return ''
@ -835,7 +835,7 @@ def slave_lag(**connection_args):
try:
_execute(cur, qry)
except MySQLdb.OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return -3
@ -920,7 +920,7 @@ def db_list(**connection_args):
try:
_execute(cur, qry)
except MySQLdb.OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return []
@ -1009,7 +1009,7 @@ def db_tables(name, **connection_args):
try:
_execute(cur, qry)
except MySQLdb.OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return []
@ -1044,7 +1044,7 @@ def db_exists(name, **connection_args):
try:
_execute(cur, qry, args)
except MySQLdb.OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return False
@ -1099,7 +1099,7 @@ def db_create(name, character_set=None, collate=None, **connection_args):
log.info('DB \'%s\' created', name)
return True
except MySQLdb.OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return False
@ -1135,7 +1135,7 @@ def db_remove(name, **connection_args):
try:
_execute(cur, qry)
except MySQLdb.OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return False
@ -1167,7 +1167,7 @@ def user_list(**connection_args):
qry = 'SELECT User,Host FROM mysql.user'
_execute(cur, qry)
except MySQLdb.OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return []
@ -1250,7 +1250,7 @@ def user_exists(user,
try:
_execute(cur, qry, args)
except MySQLdb.OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return False
@ -1282,7 +1282,7 @@ def user_info(user, host='localhost', **connection_args):
try:
_execute(cur, qry, args)
except MySQLdb.OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return False
@ -1382,7 +1382,7 @@ def user_create(user,
try:
_execute(cur, qry, args)
except MySQLdb.OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return False
@ -1497,7 +1497,7 @@ def user_chpass(user,
try:
result = _execute(cur, qry, args)
except MySQLdb.OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return False
@ -1552,7 +1552,7 @@ def user_remove(user,
try:
_execute(cur, qry, args)
except MySQLdb.OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return False
@ -1770,7 +1770,7 @@ def user_grants(user,
try:
_execute(cur, qry, args)
except MySQLdb.OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return False
@ -1884,7 +1884,7 @@ def grant_add(grant,
try:
_execute(cur, qry['qry'], qry['args'])
except (MySQLdb.OperationalError, MySQLdb.ProgrammingError) as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return False
@ -1958,7 +1958,7 @@ def grant_revoke(grant,
try:
_execute(cur, qry, args)
except MySQLdb.OperationalError as exc:
err = 'MySQL Error {0}: {1}'.format(*exc)
err = 'MySQL Error {0}: {1}'.format(*exc.args)
__context__['mysql.error'] = err
log.error(err)
return False

32
salt/modules/state.py
View file

@ -638,7 +638,7 @@ def apply_(mods=None, **kwargs):
.. code-block:: bash
salt '*' state.apply test pillar='{"foo": "bar"}'
salt '*' state.apply stuff pillar='{"foo": "bar"}'
.. note::
Values passed this way will override Pillar values set via
@ -681,11 +681,11 @@ def apply_(mods=None, **kwargs):
.. code-block:: bash
# Run the states configured in salt://test.sls (or salt://test/init.sls)
salt '*' state.apply test
# Run the states configured in salt://test.sls (or salt://test/init.sls)
# Run the states configured in salt://stuff.sls (or salt://stuff/init.sls)
salt '*' state.apply stuff
# Run the states configured in salt://stuff.sls (or salt://stuff/init.sls)
# and salt://pkgs.sls (or salt://pkgs/init.sls).
salt '*' state.apply test,pkgs
salt '*' state.apply stuff,pkgs
The following additional arguments are also accepted when applying
individual SLS files:
@ -705,7 +705,7 @@ def apply_(mods=None, **kwargs):
.. code-block:: bash
salt '*' state.apply test pillar='{"foo": "bar"}'
salt '*' state.apply stuff pillar='{"foo": "bar"}'
.. note::
Values passed this way will override Pillar values set via
@ -756,7 +756,7 @@ def apply_(mods=None, **kwargs):
.. code-block:: bash
salt '*' state.apply test localconfig=/path/to/minion.yml
salt '*' state.apply stuff localconfig=/path/to/minion.yml
sync_mods
If specified, the desired custom module types will be synced prior to
@ -764,8 +764,8 @@ def apply_(mods=None, **kwargs):
.. code-block:: bash
salt '*' state.apply test sync_mods=states,modules
salt '*' state.apply test sync_mods=all
salt '*' state.apply stuff sync_mods=states,modules
salt '*' state.apply stuff sync_mods=all
.. note::
This option is ignored when no SLS files are specified, as a
@ -793,8 +793,8 @@ def request(mods=None,
.. code-block:: bash
salt '*' state.request
salt '*' state.request test
salt '*' state.request test,pkgs
salt '*' state.request stuff
salt '*' state.request stuff,pkgs
'''
kwargs['test'] = True
ret = apply_(mods, **kwargs)
@ -930,7 +930,7 @@ def highstate(test=None, queue=False, **kwargs):
.. code-block:: bash
salt '*' state.highstate test pillar='{"foo": "bar"}'
salt '*' state.highstate stuff pillar='{"foo": "bar"}'
.. note::
Values passed this way will override Pillar values set via
@ -1110,7 +1110,7 @@ def sls(mods, test=None, exclude=None, queue=False, sync_mods=None, **kwargs):
.. code-block:: bash
salt '*' state.sls test pillar='{"foo": "bar"}'
salt '*' state.sls stuff pillar='{"foo": "bar"}'
.. note::
Values passed this way will override existing Pillar values set via
@ -1196,8 +1196,8 @@ def sls(mods, test=None, exclude=None, queue=False, sync_mods=None, **kwargs):
.. code-block:: bash
salt '*' state.sls test sync_mods=states,modules
salt '*' state.sls test sync_mods=all
salt '*' state.sls stuff sync_mods=states,modules
salt '*' state.sls stuff sync_mods=all
.. versionadded:: 2017.7.8,2018.3.3,Fluorine
@ -1773,7 +1773,7 @@ def show_low_sls(mods, test=None, queue=False, **kwargs):
.. code-block:: bash
salt '*' state.show_low_sls test pillar='{"foo": "bar"}'
salt '*' state.show_low_sls stuff pillar='{"foo": "bar"}'
.. note::
Values passed this way will override Pillar values set via

42
salt/netapi/rest_tornado/saltnado.py
View file

@ -502,6 +502,8 @@ class BaseSaltAPIHandler(tornado.web.RequestHandler): # pylint: disable=W0223
'''
# timeout all the futures
self.timeout_futures()
# clear local_client objects to disconnect event publisher's IOStream connections
del self.saltclients
def on_connection_close(self):
'''
@ -942,14 +944,27 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
'''
Dispatch local client commands
'''
# Generate jid before triggering a job to subscribe all returns from minions
chunk['jid'] = salt.utils.jid.gen_jid(self.application.opts)
# Generate jid and find all minions before triggering a job to subscribe all returns from minions
chunk['jid'] = salt.utils.jid.gen_jid(self.application.opts) if not chunk.get('jid', None) else chunk['jid']
minions = set(self.ckminions.check_minions(chunk['tgt'], chunk.get('tgt_type', 'glob')))
def subscribe_minion(minion):
salt_evt = self.application.event_listener.get_event(
self,
tag='salt/job/{}/ret/{}'.format(chunk['jid'], minion),
matcher=EventListener.exact_matcher)
syndic_evt = self.application.event_listener.get_event(
self,
tag='syndic/job/{}/ret/{}'.format(chunk['jid'], minion),
matcher=EventListener.exact_matcher)
return salt_evt, syndic_evt
# start listening for the event before we fire the job to avoid races
events = [
self.application.event_listener.get_event(self, tag='salt/job/'+chunk['jid']),
self.application.event_listener.get_event(self, tag='syndic/job/'+chunk['jid']),
]
events = []
for minion in minions:
salt_evt, syndic_evt = subscribe_minion(minion)
events.append(salt_evt)
events.append(syndic_evt)
f_call = self._format_call_run_job_async(chunk)
# fire a job off
@ -965,6 +980,12 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
pass
raise tornado.gen.Return('No minions matched the target. No command was sent, no jid was assigned.')
# get_event for missing minion
for minion in list(set(pub_data['minions']) - set(minions)):
salt_evt, syndic_evt = subscribe_minion(minion)
events.append(salt_evt)
events.append(syndic_evt)
# Map of minion_id -> returned for all minions we think we need to wait on
minions = {m: False for m in pub_data['minions']}
@ -1019,7 +1040,10 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
cancel_inflight_futures()
raise tornado.gen.Return(chunk_ret)
continue
f_result = f.result()
if f in events:
events.remove(f)
# if this is a start, then we need to add it to the pile
if f_result['tag'].endswith('/new'):
for minion_id in f_result['data']['minions']:
@ -1029,7 +1053,6 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
chunk_ret[f_result['data']['id']] = f_result['data']['return']
# clear finished event future
minions[f_result['data']['id']] = True
# if there are no more minions to wait for, then we are done
if not more_todo() and min_wait_time.done():
cancel_inflight_futures()
@ -1038,11 +1061,6 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
except TimeoutException:
pass
if f == events[0]:
events[0] = self.application.event_listener.get_event(self, tag='salt/job/'+chunk['jid'])
else:
events[1] = self.application.event_listener.get_event(self, tag='syndic/job/'+chunk['jid'])
@tornado.gen.coroutine
def job_not_running(self, jid, tgt, tgt_type, minions, is_finished):
'''

4
salt/pillar/pepa.py
View file

@ -364,8 +364,8 @@ def key_value_to_tree(data):
for flatkey, value in six.iteritems(data):
t = tree
keys = flatkey.split(__opts__['pepa_delimiter'])
for key in keys:
if key == keys[-1]:
for i, key in enumerate(keys, 1):
if i == len(keys):
t[key] = value
else:
t = t.setdefault(key, {})

4
salt/renderers/py.py
View file

@ -47,8 +47,8 @@ execution functions, grains, pillar, etc. They are:
``/srv/salt/foo/bar/baz.sls``, then ``__sls__`` in that file will be
``foo.bar.baz``.
The global context ``data`` (same as context ``{{ data }}`` for states written
with Jinja + YAML). The following YAML + Jinja state declaration:
When writing a reactor SLS file the global context ``data`` (same as context ``{{ data }}``
for states written with Jinja + YAML) is available. The following YAML + Jinja state declaration:
.. code-block:: jinja

16
salt/states/service.py
View file

@ -422,9 +422,16 @@ def running(name,
else:
before_toggle_enable_status = True
unmask_ret = {'comment': ''}
if unmask:
unmask_ret = unmasked(name, unmask_runtime)
# See if the service is already running
if before_toggle_status:
ret['comment'] = 'The service {0} is already running'.format(name)
ret['comment'] = '\n'.join(
[_f for _f in ['The service {0} is already running'.format(name),
unmask_ret['comment']] if _f]
)
if enable is True and not before_toggle_enable_status:
ret.update(_enable(name, None, **kwargs))
elif enable is False and before_toggle_enable_status:
@ -434,7 +441,9 @@ def running(name,
# Run the tests
if __opts__['test']:
ret['result'] = None
ret['comment'] = 'Service {0} is set to start'.format(name)
ret['comment'] = '\n'.join(
[_f for _f in ['Service {0} is set to start'.format(name),
unmask_ret['comment']] if _f])
return ret
# Conditionally add systemd-specific args to call to service.start
@ -494,6 +503,9 @@ def running(name,
.format(ret['comment'], init_delay)
)
if unmask:
ret['comment'] = '\n'.join([ret['comment'], unmask_ret['comment']])
return ret

54
salt/version.py
View file

@ -645,26 +645,47 @@ def system_information():
else:
return ''
version = system_version()
release = platform.release()
if platform.win32_ver()[0]:
# Get the version and release info based on the Windows Operating
# System Product Name. As long as Microsoft maintains a similar format
# this should be future proof
import win32api # pylint: disable=3rd-party-module-not-gated
server = {'Vista': '2008Server',
'7': '2008ServerR2',
'8': '2012Server',
'8.1': '2012ServerR2',
'10': '2016Server'}
# Starting with Python 2.7.12 and 3.5.2 the `platform.uname()` function
# started reporting the Desktop version instead of the Server version on
# Server versions of Windows, so we need to look those up
# So, if you find a Server Platform that's a key in the server
# dictionary, then lookup the actual Server Release.
# If this is a Server Platform then `GetVersionEx` will return a number
# greater than 1.
if win32api.GetVersionEx(1)[8] > 1 and release in server:
release = server[release]
import win32con # pylint: disable=3rd-party-module-not-gated
# Get the product name from the registry
hkey = win32con.HKEY_LOCAL_MACHINE
key = 'SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion'
value_name = 'ProductName'
reg_handle = win32api.RegOpenKey(hkey, key)
# Returns a tuple of (product_name, value_type)
product_name, _ = win32api.RegQueryValueEx(reg_handle, value_name)
version = 'Unknown'
release = ''
if 'Server' in product_name:
for item in product_name.split(' '):
# If it's all digits, then it's version
if re.match(r'\d+', item):
version = item
# If it starts with R and then numbers, it's the release
# ie: R2
if re.match(r'^R\d+$', item):
release = item
release = '{0}Server{1}'.format(version, release)
else:
for item in product_name.split(' '):
# If it's a number, decimal number, Thin or Vista, then it's the
# version
if re.match(r'^(\d+(\.\d+)?)|Thin|Vista$', item):
version = item
release = version
_, ver, sp, extra = platform.win32_ver()
version = ' '.join([release, ver, sp, extra])
else:
version = system_version()
release = platform.release()
system = [
('system', platform.system()),
@ -754,6 +775,7 @@ def msi_conformant_version():
noc = 65535
return '{}.{}.{}'.format(short_year, 20*(month-1)+bugfix, noc)
if __name__ == '__main__':
if len(sys.argv) == 2 and sys.argv[1] == 'msi':
# Building the msi requires an msi-conformant version

13
tests/integration/states/test_archive.py
View file

@ -31,6 +31,7 @@ ARCHIVE_LOCAL_TAR_SOURCE = 'file://{0}'.format(os.path.join(FILES, 'file', 'base
UNTAR_FILE = os.path.join(ARCHIVE_DIR, 'custom/README')
ARCHIVE_TAR_HASH = 'md5=7643861ac07c30fe7d2310e9f25ca514'
ARCHIVE_TAR_BAD_HASH = 'md5=d41d8cd98f00b204e9800998ecf8427e'
ARCHIVE_TAR_HASH_UPPER = 'md5=7643861AC07C30FE7D2310E9F25CA514'
class ArchiveTest(ModuleCase, SaltReturnAssertsMixin):
@ -232,6 +233,18 @@ class ArchiveTest(ModuleCase, SaltReturnAssertsMixin):
self.assertSaltFalseReturn(ret)
def test_local_archive_extracted_with_uppercase_source_hash(self):
'''
test archive.extracted with local file and bad hash
'''
ret = self.run_state('archive.extracted', name=ARCHIVE_DIR,
source=ARCHIVE_LOCAL_TAR_SOURCE, archive_format='tar',
source_hash=ARCHIVE_TAR_HASH_UPPER)
self.assertSaltTrueReturn(ret)
self._check_extracted(UNTAR_FILE)
def test_archive_extracted_with_non_base_saltenv(self):
'''
test archive.extracted with a saltenv other than `base`

29
tests/unit/pillar/test_pepa.py Normal file
View file

@ -0,0 +1,29 @@
# -*- coding: utf-8 -*-
# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
from collections import OrderedDict
# Import Salt Testing libs
from tests.support.unit import TestCase
# Import Salt Libs
import salt.pillar.pepa as pepa
class PepaPillarTestCase(TestCase):
def test_repeated_keys(self):
expected_result = {
"foo": {
"bar": {
"foo": True,
"baz": True,
},
},
}
data = OrderedDict([
('foo..bar..foo', True),
('foo..bar..baz', True),
])
result = pepa.key_value_to_tree(data)
self.assertDictEqual(result, expected_result)

12
tests/unit/states/test_service.py
View file

@ -57,7 +57,10 @@ class ServiceTestCase(TestCase, LoaderModuleMockMixin):
'name': 'salt', 'result': True},
{'changes': 'saltstack',
'comment': 'Service salt failed to start', 'name': 'salt',
'result': False}]
'result': False},
{'changes': 'saltstack',
'comment': 'Started Service salt\nService masking not available on this minion',
'name': 'salt', 'result': True, 'warnings': ["The 'unmask' argument is not supported by this platform/action"]}]
tmock = MagicMock(return_value=True)
fmock = MagicMock(return_value=False)
@ -91,6 +94,13 @@ class ServiceTestCase(TestCase, LoaderModuleMockMixin):
with patch.object(service, '_enable', MagicMock(return_value={'changes': 'saltstack'})):
self.assertDictEqual(service.running("salt", True), ret[4])
with patch.dict(service.__salt__, {'service.status': MagicMock(side_effect=[False, True]),
'service.enabled': MagicMock(side_effect=[False, True]),
'service.unmask': MagicMock(side_effect=[False, True]),
'service.start': MagicMock(return_value="stack")}):
with patch.object(service, '_enable', MagicMock(return_value={'changes': 'saltstack'})):
self.assertDictEqual(service.running("salt", True, unmask=True), ret[7])
with patch.dict(service.__opts__, {'test': True}):
with patch.dict(service.__salt__, {'service.status': tmock}):
self.assertDictEqual(service.running("salt"), ret[5])