Merge pull request #42930 from CorvinM/issue36942

Jinja line statements and comments

conf/master

@@ -569,18 +569,35 @@
 # The renderer to use on the minions to render the state data
 #renderer: yaml_jinja
 
-# The Jinja renderer can strip extra carriage returns and whitespace
-# See http://jinja.pocoo.org/docs/api/#high-level-api
-#
-# If this is set to True the first newline after a Jinja block is removed
-# (block, not variable tag!). Defaults to False, corresponds to the Jinja
-# environment init variable "trim_blocks".
-#jinja_trim_blocks: False
-#
-# If this is set to True leading spaces and tabs are stripped from the start
-# of a line to a block. Defaults to False, corresponds to the Jinja
-# environment init variable "lstrip_blocks".
-#jinja_lstrip_blocks: False
+# Default Jinja environment options for all templates except sls templates
+#jinja_env:
+#  block_start_string: '{%'
+#  block_end_string: '%}'
+#  variable_start_string: '{{'
+#  variable_end_string: '}}'
+#  comment_start_string: '{#'
+#  comment_end_string: '#}'
+#  line_statement_prefix: 
+#  line_comment_prefix: 
+#  trim_blocks: False
+#  lstrip_blocks: False
+#  newline_sequence: '\n'
+#  keep_trailing_newline: False
+
+# Jinja environment options for sls templates
+#jinja_sls_env:
+#  block_start_string: '{%'
+#  block_end_string: '%}'
+#  variable_start_string: '{{'
+#  variable_end_string: '}}'
+#  comment_start_string: '{#'
+#  comment_end_string: '#}'
+#  line_statement_prefix: 
+#  line_comment_prefix: 
+#  trim_blocks: False
+#  lstrip_blocks: False
+#  newline_sequence: '\n'
+#  keep_trailing_newline: False
 
 # The failhard option tells the minions to stop immediately after the first
 # failure detected in the state execution, defaults to False

conf/suse/master

@@ -540,18 +540,35 @@ syndic_user: salt
 # The renderer to use on the minions to render the state data
 #renderer: yaml_jinja
 
-# The Jinja renderer can strip extra carriage returns and whitespace
-# See http://jinja.pocoo.org/docs/api/#high-level-api
-#
-# If this is set to True the first newline after a Jinja block is removed
-# (block, not variable tag!). Defaults to False, corresponds to the Jinja
-# environment init variable "trim_blocks".
-#jinja_trim_blocks: False
-#
-# If this is set to True leading spaces and tabs are stripped from the start
-# of a line to a block. Defaults to False, corresponds to the Jinja
-# environment init variable "lstrip_blocks".
-#jinja_lstrip_blocks: False
+# Default Jinja environment options for all templates except sls templates
+#jinja_env:
+#  block_start_string: '{%'
+#  block_end_string: '%}'
+#  variable_start_string: '{{'
+#  variable_end_string: '}}'
+#  comment_start_string: '{#'
+#  comment_end_string: '#}'
+#  line_statement_prefix: 
+#  line_comment_prefix: 
+#  trim_blocks: False
+#  lstrip_blocks: False
+#  newline_sequence: '\n'
+#  keep_trailing_newline: False
+
+# Jinja environment options for sls templates
+#jinja_sls_env:
+#  block_start_string: '{%'
+#  block_end_string: '%}'
+#  variable_start_string: '{{'
+#  variable_end_string: '}}'
+#  comment_start_string: '{#'
+#  comment_end_string: '#}'
+#  line_statement_prefix: 
+#  line_comment_prefix: 
+#  trim_blocks: False
+#  lstrip_blocks: False
+#  newline_sequence: '\n'
+#  keep_trailing_newline: False
 
 # The failhard option tells the minions to stop immediately after the first
 # failure detected in the state execution, defaults to False

doc/ref/configuration/master.rst

@@ -1979,11 +1979,120 @@ the cloud profile or master config file, no templating will be performed.
 
     userdata_template: jinja
 
+.. conf_master:: jinja_env
+
+``jinja_env``
+-------------
+
+.. versionadded:: Oxygen
+
+Default: ``{}``
+
+jinja_env overrides the default Jinja environment options for
+**all templates except sls templates**.
+To set the options for sls templates use :conf_master:`jinja_sls_env`.
+
+.. note::
+
+    The `Jinja2 Environment documentation <http://jinja.pocoo.org/docs/api/#jinja2.Environment>`_ is the official source for the default values.
+    Not all the options listed in the jinja documentation can be overridden using :conf_master:`jinja_env` or :conf_master:`jinja_sls_env`.
+
+The default options are:
+
+.. code-block:: yaml
+
+    jinja_env:
+      block_start_string: '{%'
+      block_end_string: '%}'
+      variable_start_string: '{{'
+      variable_end_string: '}}'
+      comment_start_string: '{#'
+      comment_end_string: '#}'
+      line_statement_prefix: 
+      line_comment_prefix: 
+      trim_blocks: False
+      lstrip_blocks: False
+      newline_sequence: '\n'
+      keep_trailing_newline: False
+
+.. conf_master:: jinja_sls_env
+
+``jinja_sls_env``
+-----------------
+
+.. versionadded:: Oxygen
+
+Default: ``{}``
+
+jinja_sls_env sets the Jinja environment options for **sls templates**.
+The defaults and accepted options are exactly the same as they are
+for :conf_master:`jinja_env`.
+
+The default options are:
+
+.. code-block:: yaml
+
+    jinja_sls_env:
+      block_start_string: '{%'
+      block_end_string: '%}'
+      variable_start_string: '{{'
+      variable_end_string: '}}'
+      comment_start_string: '{#'
+      comment_end_string: '#}'
+      line_statement_prefix: 
+      line_comment_prefix: 
+      trim_blocks: False
+      lstrip_blocks: False
+      newline_sequence: '\n'
+      keep_trailing_newline: False
+
+Example using line statements and line comments to increase ease of use:
+
+If your configuration options are
+
+.. code-block:: yaml
+    jinja_sls_env:
+      line_statement_prefix: '%'
+      line_comment_prefix: '##'
+
+With these options jinja will interpret anything after a ``%`` at the start of a line (ignoreing whitespace)
+as a jinja statement and will interpret anything after a ``##`` as a comment.
+
+This allows the following more convenient syntax to be used:
+
+.. code-block:: yaml
+
+    ## (this comment will not stay once rendered)
+    # (this comment remains in the rendered template)
+    ## ensure all the formula services are running
+    % for service in formula_services:
+    enable_service_{{ serivce }}:
+      service.running:
+        name: {{ service }}
+    % endfor
+
+The following less convenient but equivalent syntax would have to
+be used if you had not set the line_statement and line_comment options:
+
+.. code-block:: yaml
+
+    {# (this comment will not stay once rendered) #}
+    # (this comment remains in the rendered template)
+    {# ensure all the formula services are running #}
+    {% for service in formula_services %}
+    enable_service_{{ service }}:
+      service.running:
+        name: {{ service }}
+    {% endfor %}
+
 .. conf_master:: jinja_trim_blocks
 
 ``jinja_trim_blocks``
 ---------------------
 
+.. deprecated:: Oxygen
+    Replaced by :conf_master:`jinja_env` and :conf_master:`jinja_sls_env`
+
 .. versionadded:: 2014.1.0
 
 Default: ``False``
@@ -2001,6 +2110,9 @@ to the Jinja environment init variable ``trim_blocks``.
 ``jinja_lstrip_blocks``
 -----------------------
 
+.. deprecated:: Oxygen
+    Replaced by :conf_master:`jinja_env` and :conf_master:`jinja_sls_env`
+
 .. versionadded:: 2014.1.0
 
 Default: ``False``

salt/config/__init__.py

@@ -890,6 +890,12 @@ VALID_OPTS = {
     # check in with their lists of expected minions before giving up
     'syndic_wait': int,
 
+    # Override Jinja environment option defaults for all templates except sls templates
+    'jinja_env': dict,
+
+    # Set Jinja environment options for sls templates
+    'jinja_sls_env': dict,
+
     # If this is set to True leading spaces and tabs are stripped from the start
     # of a line to a block.
     'jinja_lstrip_blocks': bool,
@@ -1642,6 +1648,8 @@ DEFAULT_MASTER_OPTS = {
     'winrepo_passphrase': '',
     'winrepo_refspecs': _DFLT_REFSPECS,
     'syndic_wait': 5,
+    'jinja_env': {},
+    'jinja_sls_env': {},
     'jinja_lstrip_blocks': False,
     'jinja_trim_blocks': False,
     'tcp_keepalive': True,

salt/daemons/masterapi.py

@@ -470,6 +470,8 @@ class RemoteFuncs(object):
         mopts['state_auto_order'] = self.opts['state_auto_order']
         mopts['state_events'] = self.opts['state_events']
         mopts['state_aggregate'] = self.opts['state_aggregate']
+        mopts['jinja_env'] = self.opts['jinja_env']
+        mopts['jinja_sls_env'] = self.opts['jinja_sls_env']
         mopts['jinja_lstrip_blocks'] = self.opts['jinja_lstrip_blocks']
         mopts['jinja_trim_blocks'] = self.opts['jinja_trim_blocks']
         return mopts

salt/master.py

@@ -1150,6 +1150,8 @@ class AESFuncs(object):
         mopts[u'state_auto_order'] = self.opts[u'state_auto_order']
         mopts[u'state_events'] = self.opts[u'state_events']
         mopts[u'state_aggregate'] = self.opts[u'state_aggregate']
+        mopts[u'jinja_env'] = self.opts[u'jinja_env']
+        mopts[u'jinja_sls_env'] = self.opts[u'jinja_sls_env']
         mopts[u'jinja_lstrip_blocks'] = self.opts[u'jinja_lstrip_blocks']
         mopts[u'jinja_trim_blocks'] = self.opts[u'jinja_trim_blocks']
         return mopts

salt/state.py

@@ -2838,6 +2838,8 @@ class BaseHighState(object):
             opts[u'default_top'] = mopts.get(u'default_top', opts.get(u'default_top'))
             opts[u'state_events'] = mopts.get(u'state_events')
             opts[u'state_aggregate'] = mopts.get(u'state_aggregate', opts.get(u'state_aggregate', False))
+            opts[u'jinja_env'] = mopts.get(u'jinja_env', {})
+            opts[u'jinja_sls_env'] = mopts.get(u'jinja_sls_env', {})
             opts[u'jinja_lstrip_blocks'] = mopts.get(u'jinja_lstrip_blocks', False)
             opts[u'jinja_trim_blocks'] = mopts.get(u'jinja_trim_blocks', False)
         return opts

salt/utils/templates.py

@@ -355,16 +355,40 @@ def render_jinja_tmpl(tmplstr, context, tmplpath=None):
         env_args['extensions'].append('jinja2.ext.loopcontrols')
     env_args['extensions'].append(salt.utils.jinja.SerializerExtension)
 
+    opt_jinja_env = opts.get('jinja_env', {})
+    opt_jinja_sls_env = opts.get('jinja_sls_env', {})
+
+    opt_jinja_env = opt_jinja_env if isinstance(opt_jinja_env, dict) else {}
+    opt_jinja_sls_env = opt_jinja_sls_env if isinstance(opt_jinja_sls_env, dict) else {}
+
     # Pass through trim_blocks and lstrip_blocks Jinja parameters
     # trim_blocks removes newlines around Jinja blocks
     # lstrip_blocks strips tabs and spaces from the beginning of
     # line to the start of a block.
     if opts.get('jinja_trim_blocks', False):
         log.debug('Jinja2 trim_blocks is enabled')
-        env_args['trim_blocks'] = True
+        log.warning('jinja_trim_blocks is deprecated and will be removed in a future release, please use jinja_env and/or jinja_sls_env instead')
+        opt_jinja_env['trim_blocks'] = True
+        opt_jinja_sls_env['trim_blocks'] = True
     if opts.get('jinja_lstrip_blocks', False):
         log.debug('Jinja2 lstrip_blocks is enabled')
-        env_args['lstrip_blocks'] = True
+        log.warning('jinja_lstrip_blocks is deprecated and will be removed in a future release, please use jinja_env and/or jinja_sls_env instead')
+        opt_jinja_env['lstrip_blocks'] = True
+        opt_jinja_sls_env['lstrip_blocks'] = True
+
+    def opt_jinja_env_helper(opts, optname):
+        for k, v in six.iteritems(opts):
+            k = k.lower()
+            if hasattr(jinja2.defaults, k.upper()):
+                log.debug('Jinja2 environment {0} was set to {1} by {2}'.format(k, v, optname))
+                env_args[k] = v
+            else:
+                log.warning('Jinja2 environment {0} is not recognized'.format(k))
+
+    if 'sls' in context and context['sls'] != '':
+        opt_jinja_env_helper(opt_jinja_sls_env, 'jinja_sls_env')
+    else:
+        opt_jinja_env_helper(opt_jinja_env, 'jinja_env')
 
     if opts.get('allow_undefined', False):
         jinja_env = jinja2.Environment(**env_args)

tests/unit/templates/test_jinja.py

@@ -481,6 +481,67 @@ class TestGetTemplate(TestCase):
         )
 
 
+class TestJinjaDefaultOptions(TestCase):
+
+    def __init__(self, *args, **kws):
+        TestCase.__init__(self, *args, **kws)
+        self.local_opts = {
+            'cachedir': TEMPLATES_DIR,
+            'file_client': 'local',
+            'file_ignore_regex': None,
+            'file_ignore_glob': None,
+            'file_roots': {
+                'test': [os.path.join(TEMPLATES_DIR, 'files', 'test')]
+            },
+            'pillar_roots': {
+                'test': [os.path.join(TEMPLATES_DIR, 'files', 'test')]
+            },
+            'fileserver_backend': ['roots'],
+            'hash_type': 'md5',
+            'extension_modules': os.path.join(
+                os.path.dirname(os.path.abspath(__file__)),
+                'extmods'),
+            'jinja_env': {
+                'line_comment_prefix': '##',
+                'line_statement_prefix': '%',
+            },
+        }
+        self.local_salt = {
+             'myvar': 'zero',
+             'mylist': [0, 1, 2, 3],
+        }
+
+    def test_comment_prefix(self):
+
+        template = """
+            %- set myvar = 'one'
+            ## ignored comment 1
+            {{- myvar -}}
+            {%- set myvar = 'two' %} ## ignored comment 2
+            {{- myvar }} ## ignored comment 3
+            %- if myvar == 'two':
+            %- set myvar = 'three'
+            %- endif
+            {{- myvar -}}
+            """
+        rendered = render_jinja_tmpl(template,
+                                     dict(opts=self.local_opts, saltenv='test', salt=self.local_salt))
+        self.assertEqual(rendered, u'onetwothree')
+
+    def test_statement_prefix(self):
+
+        template = """
+            {%- set mylist = ['1', '2', '3'] %}
+            %- set mylist = ['one', 'two', 'three']
+            %- for item in mylist:
+            {{- item }}
+            %- endfor
+            """
+        rendered = render_jinja_tmpl(template,
+                                     dict(opts=self.local_opts, saltenv='test', salt=self.local_salt))
+        self.assertEqual(rendered, u'onetwothree')
+
+
 class TestCustomExtensions(TestCase):
 
     def __init__(self, *args, **kws):