Update default renderer to jinja|yaml, improve renderer docs

`yaml_jinja` and all legacy renderer syntax has been transparently
translated to the render pipeline syntax (e.g `jinja|yaml`)
under-the-hood for several years now, this just makes the default
renderer adhere to the render pipeline syntax.

It also makes significant improvements to the renderer documentation.

conf/master

@@ -577,7 +577,7 @@
 #external_nodes: None
 
 # The renderer to use on the minions to render the state data
-#renderer: yaml_jinja
+#renderer: jinja|yaml
 
 # Default Jinja environment options for all templates except sls templates
 #jinja_env:
@@ -587,8 +587,8 @@
 #  variable_end_string: '}}'
 #  comment_start_string: '{#'
 #  comment_end_string: '#}'
-#  line_statement_prefix: 
-#  line_comment_prefix: 
+#  line_statement_prefix:
+#  line_comment_prefix:
 #  trim_blocks: False
 #  lstrip_blocks: False
 #  newline_sequence: '\n'
@@ -602,8 +602,8 @@
 #  variable_end_string: '}}'
 #  comment_start_string: '{#'
 #  comment_end_string: '#}'
-#  line_statement_prefix: 
-#  line_comment_prefix: 
+#  line_statement_prefix:
+#  line_comment_prefix:
 #  trim_blocks: False
 #  lstrip_blocks: False
 #  newline_sequence: '\n'

conf/minion

@@ -473,19 +473,18 @@
 
 #####    State Management Settings    #####
 ###########################################
-# The state management system executes all of the state templates on the minion
-# to enable more granular control of system state management. The type of
-# template and serialization used for state management needs to be configured
-# on the minion, the default renderer is yaml_jinja. This is a yaml file
-# rendered from a jinja template, the available options are:
-# yaml_jinja
-# yaml_mako
-# yaml_wempy
-# json_jinja
-# json_mako
-# json_wempy
+# The default renderer to use in SLS files. This is configured as a
+# pipe-delimited expression. For example, jinja|yaml will first run jinja
+# templating on the SLS file, and then load the result as YAML. This syntax is
+# documented in further depth at the following URL:
 #
-#renderer: yaml_jinja
+# https://docs.saltstack.com/en/latest/ref/renderers/#composing-renderers
+#
+# NOTE: The "shebang" prefix (e.g. "#!jinja|yaml") described in the
+# documentation linked above is for use in an SLS file to override the default
+# renderer, it should not be used when configuring the renderer here.
+#
+#renderer: jinja|yaml
 #
 # The failhard option tells the minions to stop immediately after the first
 # failure detected in the state execution. Defaults to False.

conf/proxy

@@ -356,19 +356,18 @@
 
 #####    State Management Settings    #####
 ###########################################
-# The state management system executes all of the state templates on the minion
-# to enable more granular control of system state management. The type of
-# template and serialization used for state management needs to be configured
-# on the minion, the default renderer is yaml_jinja. This is a yaml file
-# rendered from a jinja template, the available options are:
-# yaml_jinja
-# yaml_mako
-# yaml_wempy
-# json_jinja
-# json_mako
-# json_wempy
+# The default renderer to use in SLS files. This is configured as a
+# pipe-delimited expression. For example, jinja|yaml will first run jinja
+# templating on the SLS file, and then load the result as YAML. This syntax is
+# documented in further depth at the following URL:
 #
-#renderer: yaml_jinja
+# https://docs.saltstack.com/en/latest/ref/renderers/#composing-renderers
+#
+# NOTE: The "shebang" prefix (e.g. "#!jinja|yaml") described in the
+# documentation linked above is for use in an SLS file to override the default
+# renderer, it should not be used when configuring the renderer here.
+#
+#renderer: jinja|yaml
 #
 # The failhard option tells the minions to stop immediately after the first
 # failure detected in the state execution. Defaults to False.

doc/ref/configuration/master.rst

@@ -2100,13 +2100,13 @@ are enabled and available!
 ``renderer``
 ------------
 
-Default: ``yaml_jinja``
+Default: ``jinja|yaml``
 
 The renderer to use on the minions to render the state data.
 
 .. code-block:: yaml
 
-    renderer: yaml_jinja
+    renderer: jinja|json
 
 .. conf_master:: userdata_template
 

doc/ref/configuration/minion.rst

@@ -1879,13 +1879,13 @@ State Management Settings
 ``renderer``
 ------------
 
-Default: ``yaml_jinja``
+Default: ``jinja|yaml``
 
 The default renderer used for local state executions
 
 .. code-block:: yaml
 
-    renderer: yaml_jinja
+    renderer: jinja|json
 
 .. conf_minion:: test
 

doc/ref/renderers/index.rst

@@ -4,41 +4,70 @@
 Renderers
 =========
 
-The Salt state system operates by gathering information from common data
-types such as lists, dictionaries, and strings that would be familiar
-to any developer.
+The Salt state system operates by gathering information from common data types
+such as lists, dictionaries, and strings that would be familiar to any
+developer.
 
-SLS files are translated from whatever data templating format they are written
-in back into Python data types to be consumed by Salt.
+Salt Renderers translate input from the format in which it is written into
+Python data structures.
 
-By default SLS files are rendered as Jinja templates and then parsed as YAML
-documents. But since the only thing the state system cares about is raw data,
-the SLS files can be any structured format that can be dreamed up.
+The default renderer is set in the master/minion configuration file using the
+:conf_master:`renderer` config option, which defaults to ``jinja|yaml``.
 
-Currently there is support for ``Jinja + YAML``, ``Mako + YAML``,
-``Wempy + YAML``, ``Jinja + json``, ``Mako + json`` and ``Wempy + json``.
 
-Renderers can be written to support any template type. This means that the
-Salt states could be managed by XML files, HTML files, Puppet files, or any
-format that can be translated into the Pythonic data structure used by the state
-system.
+Two Kinds of Renderers
+----------------------
 
-Multiple Renderers
-------------------
+Renderers fall into one of two categories, based on what they output: text or
+data. The one exception to this would be the :mod:`pure python
+<salt.renderers.py>` renderer, which can be used in either capacity.
 
-A default renderer is selected in the master configuration file by providing
-a value to the ``renderer`` key.
+Text Renderers
+**************
 
-When evaluating an SLS, more than one renderer can be used.
+A text renderer returns text. These include templating engines such as
+:mod:`jinja <salt.renderers.jinja>`, :mod:`mako <salt.renderers.mako>`, and
+:mod:`genshi <salt.renderers.genshi>`, as well as the :mod:`gpg
+<salt.renderers.gpg>` renderer. The following are all text renderers:
 
-When rendering SLS files, Salt checks for the presence of a Salt-specific
-shebang line.
+- :mod:`aws_kms <salt.renderers.aws_kms>`
+- :mod:`cheetah <salt.renderers.cheetah>`
+- :mod:`genshi <salt.renderers.genshi>`
+- :mod:`gpg <salt.renderers.gpg>`
+- :mod:`jinja <salt.renderers.jinja>`
+- :mod:`mako <salt.renderers.mako>`
+- :mod:`nacl <salt.renderers.nacl>`
+- :mod:`pass <salt.renderers.pass>`
+- :mod:`py <salt.renderers.py>`
+- :mod:`wempy <salt.renderers.wempy>`
 
-The shebang line directly calls the name of the renderer as it is specified
-within Salt. One of the most common reasons to use multiple renderers is to
-use the Python or ``py`` renderer.
+Data Renderers
+**************
 
-Below, the first line is a shebang that references the ``py`` renderer.
+A data renderer returns a Python data structure (typically a dictionary). The
+following are all data renderers:
+
+- :mod:`dson <salt.renderers.dson>`
+- :mod:`hjson <salt.renderers.hjson>`
+- :mod:`json5 <salt.renderers.json5>`
+- :mod:`json <salt.renderers.json>`
+- :mod:`pydsl <salt.renderers.pydsl>`
+- :mod:`pyobjects <salt.renderers.pyobjects>`
+- :mod:`py <salt.renderers.py>`
+- :mod:`stateconf <salt.renderers.stateconf>`
+- :mod:`yamlex <salt.renderers.yamlex>`
+- :mod:`yaml <salt.renderers.yaml>`
+
+
+Overriding the Default Renderer
+-------------------------------
+
+It can sometimes be beneficial to write an SLS file using a renderer other than
+the default one. This can be done by using a "shebang"-like syntax on the first
+line of the SLS file:
+
+Here is an example of using the :mod:`pure python <salt.renderers.py>` renderer
+to install a package:
 
 .. code-block:: python
 
@@ -46,36 +75,57 @@ Below, the first line is a shebang that references the ``py`` renderer.
 
     def run():
         '''
-        Install the python-mako package
+        Install version 1.5-1.el7 of package "python-foo"
         '''
-        return {'include': ['python'],
-                'python-mako': {'pkg': ['installed']}}
+        return {
+            'include': ['python'],
+            'python-foo': {
+                'pkg.installed': [
+                    {'version': '1.5-1.el7'},
+                ]
+            }
+        }
 
+This would be equivalent to the following:
+
+.. code-block:: yaml
+
+    include:
+      - python
+
+    python-foo:
+      pkg.installed:
+        - version: '1.5-1.el7'
 
 .. _renderers-composing:
 
-Composing Renderers
--------------------
-A renderer can be composed from other renderers by connecting them in a series
-of pipes(``|``).
+Composing Renderers (a.k.a. The "Render Pipeline")
+--------------------------------------------------
 
-In fact, the default ``Jinja + YAML`` renderer is implemented by connecting a YAML
-renderer to a Jinja renderer. Such renderer configuration is specified as: ``jinja | yaml``.
+A render pipeline can be composed from other renderers by connecting them in a
+series of "pipes" (i.e. ``|``). The renderers will be evaluated from left to
+right, with each renderer receiving the result of the previous renderer's
+execution.
 
-Other renderer combinations are possible:
+Take for example the default renderer (``jinja|yaml``). The file is evaluated
+first a jinja template, and the result of that template is evaluated as a YAML
+document.
+
+Other render pipeline combinations include:
 
   ``yaml``
-      i.e, just YAML, no templating.
+      Just YAML, no templating.
 
-  ``mako | yaml``
-      pass the input to the ``mako`` renderer, whose output is then fed into the
-      ``yaml`` renderer.
+  ``mako|yaml``
+      This passes the input to the ``mako`` renderer, with its output fed into
+      the ``yaml`` renderer.
 
-  ``jinja | mako | yaml``
+  ``jinja|mako|yaml``
       This one allows you to use both jinja and mako templating syntax in the
       input and then parse the final rendered output as YAML.
 
-The following is a contrived example SLS file using the ``jinja | mako | yaml`` renderer:
+The following is a contrived example SLS file using the ``jinja|mako|yaml``
+render pipeline:
 
 .. code-block:: python
 
@@ -91,36 +141,59 @@ The following is a contrived example SLS file using the ``jinja | mako | yaml``
     <%doc> ${...} is Mako's notation, and so is this comment. </%doc>
     {#     Similarly, {{...}} is Jinja's notation, and so is this comment. #}
 
-For backward compatibility, ``jinja | yaml`` can also be written as
-``yaml_jinja``, and similarly, the ``yaml_mako``, ``yaml_wempy``,
-``json_jinja``, ``json_mako``, and ``json_wempy`` renderers are all supported.
+.. important::
+    Keep in mind that not all renderers can be used alone or with any other
+    renderers. For example, text renderers shouldn't be used alone as their
+    outputs are just strings, which still need to be parsed by another renderer
+    to turn them into Python data structures.
 
-Keep in mind that not all renderers can be used alone or with any other renderers.
-For example, the template renderers shouldn't be used alone as their outputs are
-just strings, which still need to be parsed by another renderer to turn them into
-highstate data structures.
+    For example, it would not make sense to use ``yaml|jinja`` because the
+    output of the :mod:`yaml <salt.renderers.yaml>` renderer is a Python data
+    structure, and the :mod:`jinja <salt.renderers.jinja>` renderer only
+    accepts text as input.
+
+    Therefore, when combining renderers, you should know what each renderer
+    accepts as input and what it returns as output. One way of thinking about
+    it is that you can chain together multiple text renderers, but the pipeline
+    *must* end in a data renderer. Similarly, since the text renderers in Salt
+    don't accept data structures as input, a text renderer should usually not
+    come after a data renderer. It's technically *possible* to write a renderer
+    that takes a data structure as input and returns a string, but no such
+    renderer is distributed with Salt.
 
-For example, it doesn't make sense to specify ``yaml | jinja`` because the
-output of the YAML renderer is a highstate data structure (a dict in Python), which
-cannot be used as the input to a template renderer. Therefore, when combining
-renderers, you should know what each renderer accepts as input and what it returns
-as output.
 
 Writing Renderers
 -----------------
 
-A custom renderer must be a Python module placed in the renderers directory and the
-module implement the ``render`` function.
+A custom renderer must be a Python module which implements a ``render``
+function. This function must implement three positional arguments:
 
-The ``render`` function will be passed the path of the SLS file as an argument.
+1. ``data`` - Can be called whatever you like. This is the input to be
+   rendered.
+2. ``saltenv``
+3. ``sls``
 
-The purpose of the ``render`` function is to parse the passed file and to return
-the Python data structure derived from the file.
+The first is the important one, and the 2nd and 3rd must be included since Salt
+needs to pass this info to each render, even though it is only used by template
+renderers.
 
-Custom renderers must be placed in a ``_renderers`` directory within the
-:conf_master:`file_roots` specified by the master config file.
+Renderers should be written so that the ``data`` argument can accept either
+strings or file-like objects as input. For example:
 
-Custom renderers are distributed when any of the following are run:
+.. code-block:: python
+
+    import mycoolmodule
+    from salt.ext import six
+
+    def render(data, saltenv='base', sls='', **kwargs):
+        if not isinstance(data, six.string_types):
+            # Read from file-like object
+            data = data.read()
+
+        return mycoolmodule.do_something(data)
+
+Custom renderers should be placed within ``salt://_renderers/``, so that they
+can be synced to minions. They are synced when any of the following are run:
 
 - :py:func:`state.apply <salt.modules.state.apply_>`
 - :py:func:`saltutil.sync_renderers <salt.modules.saltutil.sync_renderers>`
@@ -130,6 +203,12 @@ Any custom renderers which have been synced to a minion, that are named the
 same as one of Salt's default set of renderers, will take the place of the
 default renderer with the same name.
 
+.. note::
+    Renderers can also be synced from ``salt://_renderers/`` to the Master
+    using either the :py:func:`saltutil.sync_renderers
+    <salt.runners.saltutil.sync_renderers>` or :py:func:`saltutil.sync_all
+    <salt.runners.saltutil.sync_all>` runner function.
+
 
 Examples
 --------

doc/topics/jinja/index.rst

@@ -870,7 +870,7 @@ Example:
 .. note::
 
     This option may have adverse effects when using the default renderer,
-    ``yaml_jinja``. This is due to the fact that YAML requires proper handling
+    ``jinja|yaml``. This is due to the fact that YAML requires proper handling
     in regard to special characters. Please see the section on :ref:`YAML ASCII
     support <yaml_plain_ascii>` in the :ref:`YAML Idiosyncracies
     <yaml-idiosyncrasies>` documentation for more information.
@@ -1364,11 +1364,11 @@ Example:
 
 .. note::
 
-    This option may have adverse effects when using the default renderer, ``yaml_jinja``.
-    This is due to the fact that YAML requires proper handling in regard to special
-    characters. Please see the section on :ref:`YAML ASCII support <yaml_plain_ascii>`
-    in the :ref:`YAML Idiosyncracies <yaml-idiosyncrasies>` documentation for more
-    information.
+    This option may have adverse effects when using the default renderer,
+    ``jinja|yaml``. This is due to the fact that YAML requires proper handling
+    in regard to special characters. Please see the section on :ref:`YAML ASCII
+    support <yaml_plain_ascii>` in the :ref:`YAML Idiosyncracies
+    <yaml-idiosyncrasies>` documentation for more information.
 
 .. jinja_ref:: dns_check
 

doc/topics/spm/spm_formula.rst

@@ -161,7 +161,7 @@ Sections with ``pre`` in their name are evaluated before a package is installed
 and sections with ``post`` are evaluated after a package is installed. ``local``
 states are evaluated before ``tgt`` states.
 
-Each of these sections needs to be evaluated as text, rather than as YAML. 
+Each of these sections needs to be evaluated as text, rather than as YAML.
 Consider the following block:
 
 .. code-block:: text
@@ -242,7 +242,7 @@ You may also declare your own variables inside the ``FORMULA``. If SPM doesn't
 recognize them then it will ignore them, so there are no restrictions on
 variable names, outside of avoiding reserved words.
 
-By default the renderer is set to ``yaml_jinja``. You may change this by
+By default the renderer is set to ``jinja|yaml``. You may change this by
 changing the ``renderer`` setting in the ``FORMULA`` itself.
 
 Building a Package

doc/topics/tutorials/starting_states.rst

@@ -321,8 +321,8 @@ with YAML. Salt defaults to YAML because it is very straightforward and easy
 to learn and use. But the SLS files can be rendered from almost any imaginable
 medium, so long as a renderer module is provided.
 
-The default rendering system is the ``yaml_jinja`` renderer. The
-``yaml_jinja`` renderer will first pass the template through the `Jinja2`_
+The default rendering system is the ``jinja|yaml`` renderer. The
+``jinja|yaml`` renderer will first pass the template through the `Jinja2`_
 templating system, and then through the YAML parser. The benefit here is that
 full programming constructs are available when creating SLS files.
 
@@ -352,10 +352,10 @@ gives you a `"Pythonic"`_ interface to building state data.
     :ref:`MooseFS example<jinja-example-moosefs>` below.
 
 
-Getting to Know the Default - yaml_jinja
+Getting to Know the Default - jinja|yaml
 ----------------------------------------
 
-The default renderer - ``yaml_jinja``, allows for use of the jinja
+The default renderer - ``jinja|yaml``, allows for use of the jinja
 templating system. A guide to the Jinja templating system can be found here:
 http://jinja.pocoo.org/docs
 

salt/cloud/__init__.py

@@ -1786,7 +1786,7 @@ class Map(Cloud):
             else:
                 cached_map = self.opts['map']
             try:
-                renderer = self.opts.get('renderer', 'yaml_jinja')
+                renderer = self.opts.get('renderer', 'jinja|yaml')
                 rend = salt.loader.render(self.opts, {})
                 blacklist = self.opts.get('renderer_blacklist')
                 whitelist = self.opts.get('renderer_whitelist')

salt/config/__init__.py

@@ -1230,7 +1230,7 @@ DEFAULT_MINION_OPTS = {
     'sock_dir': os.path.join(salt.syspaths.SOCK_DIR, 'minion'),
     'sock_pool_size': 1,
     'backup_mode': '',
-    'renderer': 'yaml_jinja',
+    'renderer': 'jinja|yaml',
     'renderer_whitelist': [],
     'renderer_blacklist': [],
     'random_startup_delay': 0,
@@ -1665,7 +1665,7 @@ DEFAULT_MASTER_OPTS = {
     'conf_file': os.path.join(salt.syspaths.CONFIG_DIR, 'master'),
     'open_mode': False,
     'auto_accept': False,
-    'renderer': 'yaml_jinja',
+    'renderer': 'jinja|yaml',
     'renderer_whitelist': [],
     'renderer_blacklist': [],
     'failhard': False,

salt/sdb/rest.py

@@ -105,7 +105,7 @@ def query(key, value=None, service=None, profile=None):  # pylint: disable=W0613
         pair_key, pair_val = pair.split('=')
         key_vars[pair_key] = pair_val
 
-    renderer = __opts__.get('renderer', 'yaml_jinja')
+    renderer = __opts__.get('renderer', 'jinja|yaml')
     rend = salt.loader.render(__opts__, {})
     blacklist = __opts__.get('renderer_blacklist')
     whitelist = __opts__.get('renderer_whitelist')

salt/spm/__init__.py

@@ -1083,7 +1083,7 @@ class SPMClient(object):
         Render a [pre|post]_local_state or [pre|post]_tgt_state script
         '''
         # FORMULA can contain a renderer option
-        renderer = formula_def.get('renderer', self.opts.get('renderer', 'yaml_jinja'))
+        renderer = formula_def.get('renderer', self.opts.get('renderer', 'jinja|yaml'))
         rend = salt.loader.render(self.opts, {})
         blacklist = self.opts.get('renderer_blacklist')
         whitelist = self.opts.get('renderer_whitelist')

salt/state.py

@@ -2942,7 +2942,7 @@ class BaseHighState(object):
         mopts = self.client.master_opts()
         if not isinstance(mopts, dict):
             # An error happened on the master
-            opts['renderer'] = 'yaml_jinja'
+            opts['renderer'] = 'jinja|yaml'
             opts['failhard'] = False
             opts['state_top'] = salt.utils.url.create('top.sls')
             opts['nodegroups'] = {}

salt/utils/dictupdate.py

@@ -102,7 +102,7 @@ def merge_overwrite(obj_a, obj_b, merge_lists=False):
 
 def merge(obj_a, obj_b, strategy='smart', renderer='yaml', merge_lists=False):
     if strategy == 'smart':
-        if renderer == 'yamlex' or renderer.startswith('yamlex_'):
+        if renderer.split('|')[-1] == 'yamlex' or renderer.startswith('yamlex_'):
             strategy = 'aggregate'
         else:
             strategy = 'recurse'

salt/utils/http.py

@@ -820,7 +820,7 @@ def _render(template, render, renderer, template_dict, opts):
         if template_dict is None:
             template_dict = {}
         if not renderer:
-            renderer = opts.get('renderer', 'yaml_jinja')
+            renderer = opts.get('renderer', 'jinja|yaml')
         rend = salt.loader.render(opts, {})
         blacklist = opts.get('renderer_blacklist')
         whitelist = opts.get('renderer_whitelist')