Merge branch 'patch-1' of https://github.com/ITJamie/salt into patch-1

changelog/3396.added

@@ -0,0 +1,5 @@
+Introduce a `LIB_STATE_DIR` syspaths variable which defaults to `CONFIG_DIR`,
+but can be individually customized during installation by specifying
+`--salt-lib-state-dir` during installation. Change the default `pki_dir` to
+`<LIB_STATE_DIR>/pki/master` (for the master) and `<LIB_STATE_DIR>/pki/minion`
+(for the minion).

changelog/39292.fixed

@@ -0,0 +1 @@
+Fix mounted bind mounts getting active mount options added

changelog/40054.fixed

@@ -0,0 +1 @@
+Fix `sysctl.present` converts spaces to tabs.

changelog/53988.fixed

@@ -0,0 +1 @@
+Avoid warning noise in lograte.get

changelog/54508.fixed

@@ -0,0 +1 @@
+Fix mount.mounted does not handle blanks properly

changelog/54682.fixed

@@ -0,0 +1 @@
+Fixed grain num_cpus get wrong CPUs count in case of inconsistent CPU numbering.

changelog/56016.fixed

@@ -0,0 +1 @@
+Included stdout in error message for Zypper calls in zypperpkg module.

changelog/58297.fixed

@@ -0,0 +1 @@
+Add missing MariaDB Grants to mysql module. MariaDB has added some grants in 10.4.x and 10.5.x that are not present here, which results in an error when creating.

changelog/59439.changed

@@ -0,0 +1 @@
+Improve support for listing macOS brew casks

changelog/60227.fixed

@@ -0,0 +1 @@
+Update docs remove dry_run in docstring of file.blockreplace state.

changelog/60467.fixed

@@ -0,0 +1 @@
+Use return code in iptables --check to verify rule exists.

changelog/60722.added

@@ -0,0 +1 @@
+Added .0 back to our versioning scheme for future versions (e.g. 3006.0)

changelog/60853.fixed

@@ -0,0 +1 @@
+Fix xfs module when additional output included in mkfs.xfs command.

changelog/61827.fixed

@@ -0,0 +1 @@
+Ignore errors on reading license info with dpkg_lowpkg to prevent tracebacks on getting package information.

changelog/61859.fixed

@@ -0,0 +1 @@
+win_lgpo: Display conflicting policy names when more than one policy is found

changelog/61860.fixed

@@ -0,0 +1 @@
+win_lgpo: Fixed intermittent KeyError when getting policy setting using lgpo.get_policy

changelog/61966.fixed

@@ -0,0 +1 @@
+Fixed listing minions on OpenBSD

changelog/62058.fixed

@@ -0,0 +1 @@
+Fixed issue with some LGPO policies having whitespace at the beginning or end of the element alias

changelog/62120.added

@@ -0,0 +1,4 @@
+Config option pass_variable_prefix allows to distinguish variables that contain paths to pass secrets.
+Config option pass_strict_fetch allows to error out when a secret cannot be fetched from pass.
+Config option pass_dir allows setting the PASSWORD_STORE_DIR env for pass.
+Config option pass_gnupghome allows setting the $GNUPGHOME env for pass.

changelog/62120.fixed

@@ -0,0 +1,4 @@
+Pass executable path from _get_path_exec() is used when calling the program.
+The $HOME env is no longer modified globally.
+Only trailing newlines are stripped from the fetched secret.
+Pass process arguments are handled in a secure way.

changelog/62198.fixed

@@ -0,0 +1 @@
+Recognize OSMC as Debian-based

changelog/62211.fixed

@@ -0,0 +1 @@
+Improved error handling and diagnostics in the proxmox salt-cloud driver

changelog/62273.fixed

@@ -0,0 +1 @@
+Fix localhost detection in glusterfs.peers

changelog/62283.fixed

@@ -0,0 +1 @@
+Fix matcher slowness due to loader invocation

changelog/62323.fixed

@@ -0,0 +1 @@
+Fixes the Puppet module for non-aio Puppet packages for example running the Puppet module on FreeBSD.

changelog/62346.changed

@@ -0,0 +1 @@
+Enhance capture of error messages for Zypper calls in zypperpkg module.

changelog/62398.fixed

@@ -0,0 +1 @@
+Fix ability to execute remote file client methods in saltcheck

changelog/62426.fixed

@@ -0,0 +1 @@
+Added directory mode for file.copy with makedirs

changelog/62439.fixed

@@ -0,0 +1 @@
+When handling aggregation, change the order to ensure that the requisites are aggregated first and then the state functions are aggregated.  Caching whether aggregate functions are available for particular states so we don't need to attempt to load them everytime.

changelog/62461.fixed

@@ -0,0 +1 @@
+sysctl.persist now updates the in-memory value on FreeBSD even if the on-disk value was already correct.

changelog/62499.fixed

@@ -0,0 +1 @@
+Fix user/group checking on file state functions in the test mode.

changelog/62508.added

@@ -0,0 +1 @@
+Add ifelse Jinja function as found in CFEngine

changelog/62521.fixed

@@ -0,0 +1 @@
+Fixed the omitted "pool" parameter when cloning a VM with the proxmox salt-cloud driver

changelog/62523.fixed

@@ -0,0 +1 @@
+Fix rendering of pyobjects states in saltcheck

changelog/62539.added

@@ -0,0 +1 @@
+Implementation of Amazon EC2 instance detection and setting `virtual_subtype` grain accordingly including the product if possible to identify.

changelog/62547.fixed

@@ -0,0 +1 @@
+Remove the connection_timeout from netmiko_connection_args before netmiko_connection_args is added to __context__["netmiko_device"]["args"] which is passed along to the Netmiko library.

changelog/62558.fixed

@@ -0,0 +1 @@
+Fixed salt-cloud cloning a proxmox VM with a specified new vmid.

changelog/62578.added

@@ -0,0 +1,2 @@
+Adds __env__substitution to ext_pillar.stack; followup of #61531, improved exception handling for stacked template (jinja) template rendering and yaml parsing in ext_pillar.stack
+

changelog/62580.fixed

@@ -0,0 +1 @@
+Fixed missing parameters when cloning a VM with the proxmox salt-cloud driver

changelog/62595.fixed

@@ -0,0 +1 @@
+Correctly reraise exceptions in states.http

changelog/62624.fixed

@@ -0,0 +1 @@
+updated rest_cherry/app to properly detect arg sent as a string as curl will do when only one arg is supplied. 

changelog/62633.fixed

@@ -0,0 +1 @@
+Prevent possible tracebacks in core grains module by ignoring non utf8 characters in /proc/1/environ, /proc/1/cmdline, /proc/cmdline

changelog/62651.fixed

@@ -0,0 +1 @@
+Fixed vault ext pillar return data for KV v2

changelog/62657.fixed

@@ -0,0 +1 @@
+Fix groupadd.* functions hard code relative command name

changelog/62670.fixed

@@ -0,0 +1 @@
+Fixed pdbedit.create trying to use a bytes-like hash as string.

changelog/62678.added

@@ -0,0 +1 @@
+Increase file.tidied flexibility with regard to age and size

changelog/62728.fixed

@@ -0,0 +1 @@
+Change startup ReqServer log messages from error to info level.

changelog/62761.added

@@ -0,0 +1 @@
+Added "connected_devices" feature to netbox pillar module. It contains extra information about devices connected to the minion

changelog/62768.added

@@ -0,0 +1 @@
+Add atomic file operation for symlink changes

changelog/62772.fixed

@@ -0,0 +1 @@
+Fix kmod.* functions hard code relative command name

doc/ref/configuration/master.rst

@@ -180,10 +180,15 @@ The path to the master's configuration file.
 ``pki_dir``
 -----------
 
-Default: ``/etc/salt/pki/master``
+Default: ``<LIB_STATE_DIR>/pki/master``
 
 The directory to store the pki authentication keys.
 
+``<LIB_STATE_DIR>`` is the pre-configured variable state directory set during
+installation via ``--salt-lib-state-dir``. It defaults to ``/etc/salt``. Systems
+following the Filesystem Hierarchy Standard (FHS) might set it to
+``/var/lib/salt``.
+
 .. code-block:: yaml
 
     pki_dir: /etc/salt/pki/master
@@ -2646,14 +2651,14 @@ Enable extra routines for YAML renderer used states containing UTF characters.
 ``runner_returns``
 ------------------
 
-Default: ``False``
+Default: ``True``
 
-If set to ``True``, runner jobs will be saved to job cache (defined by
+If set to ``False``, runner jobs will not be saved to job cache (defined by
 :conf_master:`master_job_cache`).
 
 .. code-block:: yaml
 
-    runner_returns: True
+    runner_returns: False
 
 
 .. _master-file-server-settings:

doc/ref/configuration/minion.rst

@@ -602,10 +602,15 @@ The path to the minion's configuration file.
 ``pki_dir``
 -----------
 
-Default: ``/etc/salt/pki/minion``
+Default: ``<LIB_STATE_DIR>/pki/minion``
 
 The directory used to store the minion's public and private keys.
 
+``<LIB_STATE_DIR>`` is the pre-configured variable state directory set during
+installation via ``--salt-lib-state-dir``. It defaults to ``/etc/salt``. Systems
+following the Filesystem Hierarchy Standard (FHS) might set it to
+``/var/lib/salt``.
+
 .. code-block:: yaml
 
     pki_dir: /etc/salt/pki/minion

doc/security/index.rst

@@ -143,11 +143,16 @@ quickly and safely as is possible.
 4.  Packagers are notified via the `salt-packagers`_ mailing list that an issue
     was reported and resolved, and that an announcement is incoming.
 
-5.  A new release is created and pushed to all affected repositories. The
+5.  A pre-announcement is sent out to the `salt-announce`_ mailing list approximately
+    a week before the CVE release. This announcement does not include details
+    of the vulnerability. The pre-announcement will include the date the release
+    will occur and the vulnerability rating.
+
+6.  A new release is created and pushed to all affected repositories. The
     release documentation provides a full description of the issue, plus any
     upgrade instructions or other relevant details.
 
-6.  An announcement is made to the `salt-users`_ and `salt-announce`_ mailing
+7.  An announcement is made to the `salt-users`_ and `salt-announce`_ mailing
     lists. The announcement contains a description of the issue and a link to
     the full release documentation and download locations.
 

doc/topics/development/tests/test-pipeline.rst

@@ -0,0 +1,32 @@
+.. _test-pipeline:
+
+=============
+Test Pipeline
+=============
+
+Salt's test suite is run on `jenkins`_. We have added the ``@pytest.mark.slow_test``
+pytest decorator to help designate tests that take a significant amount of time to
+run. These tests are only run on our branch tests, unless your PR is editing code
+that requires running a specific slow test. When you submit a PR it will by default,
+only run the tests that do not include the ``@pytest.mark.slow_test`` pytest decorator.
+
+
+Process to Fix Test Failures on Branch Tests
+--------------------------------------------
+
+If there is a failure on the branch tests on `jenkins`_, this is the process to follow
+to ensure it is fixed.
+
+- Review the issues in Salt repo with the label ``Test-Failure`` to ensure there isn't
+  an already open issue assigned to someone to fix.
+- If there is not an issue open for the failing test, create a new issue in Salt's repo
+- Select "Test Failure" and the issue will create the correct template you need.
+- Include the name of the test that is failing in the title
+- Include the jenkins URL to the test in the body and any additional information needed.
+- When you create the issue it will automatically add the label ``Test-Failure``.
+- If you are going to fix the test assign yourself to the issue.
+- If you are not going to fix the test, there is nothing else to do. The core team will
+  review these open issues and ensure they are assinged out to be fixed.
+
+
+.. _jenkins: https://jenkins.saltproject.io

doc/topics/jinja/index.rst

@@ -2299,6 +2299,41 @@ will be rendered as:
 
   unique = ['foo', 'bar']
 
+Global Functions
+================
+
+Salt Project extends `builtin global functions`_ with these custom global functions:
+
+.. jinja_ref:: ifelse
+
+``ifelse``
+----------
+
+Evaluate each pair of arguments up to the last one as a (matcher, value)
+tuple, returning ``value`` if matched.  If none match, returns the last
+argument.
+
+The ``ifelse`` function is like a multi-level if-else statement. It was
+inspired by CFEngine's ``ifelse`` function which in turn was inspired by
+Oracle's ``DECODE`` function. It must have an odd number of arguments (from
+1 to N). The last argument is the default value, like the ``else`` clause in
+standard programming languages. Every pair of arguments before the last one
+are evaluated as a pair. If the first one evaluates true then the second one
+is returned, as if you had used the first one in a compound match
+expression. Boolean values can also be used as the first item in a pair, as it
+will be translated to a match that will always match ("*") or never match
+("SALT_IFELSE_MATCH_NOTHING") a target system.
+
+This is essentially another way to express the ``match.filter_by`` functionality
+in way that's familiar to CFEngine or Oracle users. Consider using
+``match.filter_by`` unless this function fits your workflow.
+
+.. code-block:: jinja
+
+    {{ ifelse('foo*', 'fooval', 'bar*', 'barval', 'defaultval', minion_id='bar03') }}
+
+.. _`builtin global functions`: https://jinja.palletsprojects.com/en/2.11.x/templates/#builtin-globals
+
 Jinja in Files
 ==============
 

doc/topics/reactor/index.rst

@@ -629,7 +629,7 @@ will be automatically accepted and have :py:func:`state.apply
 <salt.modules.state.apply_>` executed. On top of this, we're going to add that
 a host coming up that was replaced (meaning a new key) will also be accepted.
 
-Our master configuration will be rather simple. All minions that attempte to
+Our master configuration will be rather simple. All minions that attempt to
 authenticate will match the :strong:`tag` of :strong:`salt/auth`. When it comes
 to the minion key being accepted, we get a more refined :strong:`tag` that
 includes the minion id, which we can use for matching.

doc/topics/tutorials/gitfs.rst

@@ -938,7 +938,7 @@ URL using the format ``https://<user>:<password>@<url>``, like so:
     gitfs_remotes:
       - https://git:mypassword@domain.tld/myrepo.git
 
-The other way would be to configure the authentication in ``~/.netrc``:
+The other way would be to configure the authentication in ``/var/lib/salt/.netrc``:
 
 .. code-block:: text
 

requirements/static/ci/py3.10/cloud.txt

@@ -484,7 +484,7 @@ lxml==4.9.1
     # via
     #   junos-eznc
     #   ncclient
-mako==1.1.5
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.10/darwin.txt

@@ -485,7 +485,7 @@ lxml==4.9.1
     # via
     #   junos-eznc
     #   ncclient
-mako==1.1.4
+mako==1.2.2
     # via
     #   -r requirements/darwin.txt
     #   -r requirements/static/ci/common.in

requirements/static/ci/py3.10/docs.txt

@@ -493,7 +493,7 @@ lxml==4.9.1
     # via
     #   junos-eznc
     #   ncclient
-mako==1.1.5
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.10/freebsd.txt

@@ -482,7 +482,7 @@ lxml==4.9.1
     # via
     #   junos-eznc
     #   ncclient
-mako==1.1.4
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.10/lint.txt

@@ -487,7 +487,7 @@ lxml==4.9.1
     # via
     #   junos-eznc
     #   ncclient
-mako==1.1.5
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.10/linux.txt

@@ -499,7 +499,7 @@ lxml==4.9.1
     # via
     #   junos-eznc
     #   ncclient
-mako==1.1.4
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.7/cloud.txt

@@ -455,6 +455,7 @@ importlib-metadata==4.8.1 ; python_version >= "3.6" and python_version < "3.10"
     #   backports.entry-points-selectable
     #   click
     #   jsonschema
+    #   mako
     #   moto
     #   pluggy
     #   pytest
@@ -510,7 +511,7 @@ lxml==4.9.1
     #   junos-eznc
     #   napalm
     #   ncclient
-mako==1.1.5
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.7/docs.txt

@@ -463,6 +463,7 @@ importlib-metadata==4.6.4 ; python_version >= "3.6" and python_version < "3.10"
     #   backports.entry-points-selectable
     #   click
     #   jsonschema
+    #   mako
     #   moto
     #   virtualenv
 ipaddress==1.0.23
@@ -517,7 +518,7 @@ lxml==4.9.1
     #   junos-eznc
     #   napalm
     #   ncclient
-mako==1.1.5
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.7/freebsd.txt

@@ -454,6 +454,7 @@ importlib-metadata==4.6.4 ; python_version >= "3.6" and python_version < "3.10"
     #   -r requirements/static/pkg/freebsd.in
     #   backports.entry-points-selectable
     #   jsonschema
+    #   mako
     #   moto
     #   pluggy
     #   pytest
@@ -507,7 +508,7 @@ lxml==4.9.1
     #   junos-eznc
     #   napalm
     #   ncclient
-mako==1.1.4
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.7/lint.txt

@@ -457,6 +457,7 @@ importlib-metadata==4.6.4 ; python_version >= "3.6" and python_version < "3.10"
     #   backports.entry-points-selectable
     #   click
     #   jsonschema
+    #   mako
     #   moto
     #   virtualenv
 ipaddress==1.0.23
@@ -513,7 +514,7 @@ lxml==4.9.1
     #   junos-eznc
     #   napalm
     #   ncclient
-mako==1.1.5
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.7/linux.txt

@@ -468,6 +468,7 @@ importlib-metadata==4.6.4 ; python_version >= "3.6" and python_version < "3.10"
     #   -r requirements/static/pkg/linux.in
     #   backports.entry-points-selectable
     #   jsonschema
+    #   mako
     #   moto
     #   pluggy
     #   pytest
@@ -522,7 +523,7 @@ lxml==4.9.1
     #   junos-eznc
     #   napalm
     #   ncclient
-mako==1.1.4
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.7/windows.txt

@@ -131,6 +131,7 @@ importlib-metadata==4.6.4 ; python_version >= "3.6" and python_version < "3.10"
     #   -r requirements/windows.txt
     #   backports.entry-points-selectable
     #   jsonschema
+    #   mako
     #   moto
     #   pluggy
     #   pytest
@@ -171,7 +172,7 @@ libnacl==1.8.0
     # via -r requirements/windows.txt
 lxml==4.9.1
     # via -r requirements/windows.txt
-mako==1.1.4
+mako==1.2.2
     # via
     #   -r requirements/static/ci/common.in
     #   -r requirements/windows.txt

requirements/static/ci/py3.8/cloud.txt

@@ -500,7 +500,7 @@ lxml==4.9.1
     #   junos-eznc
     #   napalm
     #   ncclient
-mako==1.1.5
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.8/docs.txt

@@ -509,7 +509,7 @@ lxml==4.9.1
     #   junos-eznc
     #   napalm
     #   ncclient
-mako==1.1.5
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.8/freebsd.txt

@@ -498,7 +498,7 @@ lxml==4.9.1
     #   junos-eznc
     #   napalm
     #   ncclient
-mako==1.1.4
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.8/lint.txt

@@ -505,7 +505,7 @@ lxml==4.9.1
     #   junos-eznc
     #   napalm
     #   ncclient
-mako==1.1.5
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.8/linux.txt

@@ -513,7 +513,7 @@ lxml==4.9.1
     #   junos-eznc
     #   napalm
     #   ncclient
-mako==1.1.4
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.8/windows.txt

@@ -160,7 +160,7 @@ libnacl==1.8.0
     # via -r requirements/windows.txt
 lxml==4.9.1
     # via -r requirements/windows.txt
-mako==1.1.4
+mako==1.2.2
     # via
     #   -r requirements/static/ci/common.in
     #   -r requirements/windows.txt

requirements/static/ci/py3.9/cloud.txt

@@ -500,7 +500,7 @@ lxml==4.9.1
     #   junos-eznc
     #   napalm
     #   ncclient
-mako==1.1.5
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.9/darwin.txt

@@ -501,7 +501,7 @@ lxml==4.9.1
     #   junos-eznc
     #   napalm
     #   ncclient
-mako==1.1.4
+mako==1.2.2
     # via
     #   -r requirements/darwin.txt
     #   -r requirements/static/ci/common.in

requirements/static/ci/py3.9/docs.txt

@@ -507,7 +507,7 @@ lxml==4.9.1
     #   junos-eznc
     #   napalm
     #   ncclient
-mako==1.1.5
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.9/freebsd.txt

@@ -498,7 +498,7 @@ lxml==4.9.1
     #   junos-eznc
     #   napalm
     #   ncclient
-mako==1.1.4
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.9/lint.txt

@@ -503,7 +503,7 @@ lxml==4.9.1
     #   junos-eznc
     #   napalm
     #   ncclient
-mako==1.1.5
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.9/linux.txt

@@ -515,7 +515,7 @@ lxml==4.9.1
     #   junos-eznc
     #   napalm
     #   ncclient
-mako==1.1.4
+mako==1.2.2
     # via -r requirements/static/ci/common.in
 markupsafe==2.0.1
     # via

requirements/static/ci/py3.9/windows.txt

@@ -160,7 +160,7 @@ libnacl==1.8.0
     # via -r requirements/windows.txt
 lxml==4.9.1
     # via -r requirements/windows.txt
-mako==1.1.4
+mako==1.2.2
     # via
     #   -r requirements/static/ci/common.in
     #   -r requirements/windows.txt

requirements/static/pkg/py3.10/darwin.txt

@@ -49,7 +49,7 @@ jinja2==3.1.0
     # via -r requirements/base.txt
 linode-python==1.1.1
     # via -r requirements/darwin.txt
-mako==1.1.4
+mako==1.2.2
     # via -r requirements/darwin.txt
 markupsafe==2.0.1
     # via

requirements/static/pkg/py3.7/windows.txt

@@ -35,7 +35,9 @@ idna==2.8
 immutables==0.15
     # via contextvars
 importlib-metadata==4.6.4 ; python_version >= "3.6" and python_version < "3.10"
-    # via -r requirements/windows.txt
+    # via
+    #   -r requirements/windows.txt
+    #   mako
 ioloop==0.1a0
     # via -r requirements/windows.txt
 jaraco.classes==3.2.1
@@ -55,7 +57,7 @@ libnacl==1.8.0
     # via -r requirements/windows.txt
 lxml==4.9.1
     # via -r requirements/windows.txt
-mako==1.1.4
+mako==1.2.2
     # via -r requirements/windows.txt
 markupsafe==2.0.1
     # via

requirements/static/pkg/py3.8/windows.txt

@@ -55,7 +55,7 @@ libnacl==1.8.0
     # via -r requirements/windows.txt
 lxml==4.9.1
     # via -r requirements/windows.txt
-mako==1.1.4
+mako==1.2.2
     # via -r requirements/windows.txt
 markupsafe==2.0.1
     # via

requirements/static/pkg/py3.9/darwin.txt

@@ -51,7 +51,7 @@ jinja2==3.1.0
     # via -r requirements/base.txt
 linode-python==1.1.1
     # via -r requirements/darwin.txt
-mako==1.1.4
+mako==1.2.2
     # via -r requirements/darwin.txt
 markupsafe==2.0.1
     # via

requirements/static/pkg/py3.9/windows.txt

@@ -55,7 +55,7 @@ libnacl==1.8.0
     # via -r requirements/windows.txt
 lxml==4.9.1
     # via -r requirements/windows.txt
-mako==1.1.4
+mako==1.2.2
     # via -r requirements/windows.txt
 markupsafe==2.0.1
     # via

salt/beacons/watchdog.py

@@ -191,6 +191,7 @@ def beacon(config):
 
     The mask list can contain the following events (the default mask is create,
     modify delete, and move):
+
     * create  - File or directory is created in watched directory
     * modify  - The watched directory is modified
     * delete  - File or directory is deleted from watched directory

salt/cloud/clouds/proxmox.py

@@ -31,6 +31,7 @@ import pprint
 import re
 import socket
 import time
+import urllib
 
 import salt.config as config
 import salt.utils.cloud
@@ -136,7 +137,9 @@ def _authenticate():
     connect_data = {"username": username, "password": passwd}
     full_url = "https://{}:{}/api2/json/access/ticket".format(url, port)
 
-    returned_data = requests.post(full_url, verify=verify_ssl, data=connect_data).json()
+    response = requests.post(full_url, verify=verify_ssl, data=connect_data)
+    response.raise_for_status()
+    returned_data = response.json()
 
     ticket = {"PVEAuthCookie": returned_data["data"]["ticket"]}
     csrf = str(returned_data["data"]["CSRFPreventionToken"])
@@ -190,7 +193,12 @@ def query(conn_type, option, post_data=None):
     elif conn_type == "get":
         response = requests.get(full_url, verify=verify_ssl, cookies=ticket)
 
-    response.raise_for_status()
+    try:
+        response.raise_for_status()
+    except requests.exceptions.RequestException:
+        # Log the details of the response.
+        log.error("Error in %s query to %s:\n%s", conn_type, full_url, response.text)
+        raise
 
     try:
         returned_data = response.json()
@@ -558,20 +566,18 @@ def _reconfigure_clone(vm_, vmid):
         log.warning("Reconfiguring clones is only available under `qemu`")
         return
 
-    # TODO: Support other settings here too as these are not the only ones that can be modified after a clone operation
+    # Determine which settings can be reconfigured.
+    query_path = "nodes/{}/qemu/{}/config"
+    valid_settings = set(_get_properties(query_path.format("{node}", "{vmid}"), "POST"))
+
     log.info("Configuring cloned VM")
 
     # Modify the settings for the VM one at a time so we can see any problems with the values
     # as quickly as possible
     for setting in vm_:
-        if re.match(r"^(ide|sata|scsi)(\d+)$", setting):
-            postParams = {setting: vm_[setting]}
-            query(
-                "post",
-                "nodes/{}/qemu/{}/config".format(vm_["host"], vmid),
-                postParams,
-            )
-
+        postParams = None
+        if setting == "vmid":
+            pass  # vmid gets passed in the URL and can't be reconfigured
         elif re.match(r"^net(\d+)$", setting):
             # net strings are a list of comma seperated settings. We need to merge the settings so that
             # the setting in the profile only changes the settings it touches and the other settings
@@ -591,6 +597,13 @@ def _reconfigure_clone(vm_, vmid):
 
             # Convert the dictionary back into a string list
             postParams = {setting: _dictionary_to_stringlist(new_setting)}
+
+        elif setting == "sshkeys":
+            postParams = {setting: urllib.parse.quote(vm_[setting], safe="")}
+        elif setting in valid_settings:
+            postParams = {setting: vm_[setting]}
+
+        if postParams:
             query(
                 "post",
                 "nodes/{}/qemu/{}/config".format(vm_["host"], vmid),
@@ -655,12 +668,18 @@ def create(vm_):
         newid = _get_next_vmid()
         data = create_node(vm_, newid)
     except Exception as exc:  # pylint: disable=broad-except
+        msg = str(exc)
+        if (
+            isinstance(exc, requests.exceptions.RequestException)
+            and exc.response is not None
+        ):
+            msg = msg + "\n" + exc.response.text
         log.error(
             "Error creating %s on PROXMOX\n\n"
             "The following exception was thrown when trying to "
             "run the initial deployment: \n%s",
             vm_["name"],
-            exc,
+            msg,
             # Show the traceback if the debug logging level is enabled
             exc_info_on_loglevel=logging.DEBUG,
         )
@@ -668,10 +687,7 @@ def create(vm_):
 
     ret["creation_data"] = data
     name = vm_["name"]  # hostname which we know
-    if "clone" in vm_ and vm_["clone"] is True:
-        vmid = newid
-    else:
-        vmid = data["vmid"]  # vmid which we have received
+    vmid = data["vmid"]  # vmid which we have received
     host = data["node"]  # host which we have received
     nodeType = data["technology"]  # VM tech (Qemu / OpenVZ)
 
@@ -1007,6 +1023,7 @@ def create_node(vm_, newid):
         )
         for prop in _get_properties("/nodes/{node}/qemu", "POST", static_props):
             if prop in vm_:  # if the property is set, use it for the VM request
+                # If specified, vmid will override newid.
                 newnode[prop] = vm_[prop]
 
     # The node is ready. Lets request it to be added
@@ -1026,6 +1043,8 @@ def create_node(vm_, newid):
     if "clone" in vm_ and vm_["clone"] is True and vm_["technology"] == "qemu":
         postParams = {}
         postParams["newid"] = newnode["vmid"]
+        if "pool" in vm_:
+            postParams["pool"] = vm_["pool"]
 
         for prop in "description", "format", "full", "name":
             if (
@@ -1047,7 +1066,12 @@ def create_node(vm_, newid):
         )
     else:
         node = query("post", "nodes/{}/{}".format(vmhost, vm_["technology"]), newnode)
-    return _parse_proxmox_upid(node, vm_)
+    result = _parse_proxmox_upid(node, vm_)
+
+    # When cloning, the upid contains the clone_from vmid instead of the new vmid
+    result["vmid"] = newnode["vmid"]
+
+    return result
 
 
 def show_instance(name, call=None):

salt/config/__init__.py

@@ -964,6 +964,14 @@ VALID_OPTS = immutabletypes.freeze(
         # The port to be used when checking if a master is connected to a
         # minion
         "remote_minions_port": int,
+        # pass renderer: Fetch secrets only for the template variables matching the prefix
+        "pass_variable_prefix": str,
+        # pass renderer: Whether to error out when unable to fetch a secret
+        "pass_strict_fetch": bool,
+        # pass renderer: Set GNUPGHOME env for Pass
+        "pass_gnupghome": str,
+        # pass renderer: Set PASSWORD_STORE_DIR env for Pass
+        "pass_dir": str,
     }
 )
 
@@ -991,7 +999,7 @@ DEFAULT_MINION_OPTS = immutabletypes.freeze(
         "syndic_finger": "",
         "user": salt.utils.user.get_user(),
         "root_dir": salt.syspaths.ROOT_DIR,
-        "pki_dir": os.path.join(salt.syspaths.CONFIG_DIR, "pki", "minion"),
+        "pki_dir": os.path.join(salt.syspaths.LIB_STATE_DIR, "pki", "minion"),
         "id": "",
         "id_function": {},
         "cachedir": os.path.join(salt.syspaths.CACHE_DIR, "minion"),
@@ -1286,7 +1294,7 @@ DEFAULT_MASTER_OPTS = immutabletypes.freeze(
         "keep_jobs": 24,
         "archive_jobs": False,
         "root_dir": salt.syspaths.ROOT_DIR,
-        "pki_dir": os.path.join(salt.syspaths.CONFIG_DIR, "pki", "master"),
+        "pki_dir": os.path.join(salt.syspaths.LIB_STATE_DIR, "pki", "master"),
         "key_cache": "",
         "cachedir": os.path.join(salt.syspaths.CACHE_DIR, "master"),
         "file_roots": {
@@ -1604,6 +1612,10 @@ DEFAULT_MASTER_OPTS = immutabletypes.freeze(
         "fips_mode": False,
         "detect_remote_minions": False,
         "remote_minions_port": 22,
+        "pass_variable_prefix": "",
+        "pass_strict_fetch": False,
+        "pass_gnupghome": "",
+        "pass_dir": "",
     }
 )
 
@@ -1635,7 +1647,7 @@ DEFAULT_PROXY_MINION_OPTS = immutabletypes.freeze(
         "proxy_always_alive": True,
         "proxy_keep_alive": True,  # by default will try to keep alive the connection
         "proxy_keep_alive_interval": 1,  # frequency of the proxy keepalive in minutes
-        "pki_dir": os.path.join(salt.syspaths.CONFIG_DIR, "pki", "proxy"),
+        "pki_dir": os.path.join(salt.syspaths.LIB_STATE_DIR, "pki", "proxy"),
         "cachedir": os.path.join(salt.syspaths.CACHE_DIR, "proxy"),
         "sock_dir": os.path.join(salt.syspaths.SOCK_DIR, "proxy"),
     }

salt/matchers/compound_match.py

@@ -18,6 +18,14 @@ except ImportError:
 log = logging.getLogger(__name__)
 
 
+def _load_matchers(opts):
+    """
+    Store matchers in __context__ so they're only loaded once
+    """
+    __context__["matchers"] = {}
+    __context__["matchers"] = salt.loader.matchers(opts)
+
+
 def match(tgt, opts=None, minion_id=None):
     """
     Runs the compound target check
@@ -25,7 +33,8 @@ def match(tgt, opts=None, minion_id=None):
     if not opts:
         opts = __opts__
     nodegroups = opts.get("nodegroups", {})
-    matchers = salt.loader.matchers(opts)
+    if "matchers" not in __context__:
+        _load_matchers(opts)
     if not minion_id:
         minion_id = opts.get("id")
 
@@ -103,7 +112,7 @@ def match(tgt, opts=None, minion_id=None):
 
             results.append(
                 str(
-                    matchers["{}_match.match".format(engine)](
+                    __context__["matchers"]["{}_match.match".format(engine)](
                         *engine_args, **engine_kwargs
                     )
                 )
@@ -111,7 +120,9 @@ def match(tgt, opts=None, minion_id=None):
 
         else:
             # The match is not explicitly defined, evaluate it as a glob
-            results.append(str(matchers["glob_match.match"](word, opts, minion_id)))
+            results.append(
+                str(__context__["matchers"]["glob_match.match"](word, opts, minion_id))
+            )
 
     results = " ".join(results)
     log.debug('compound_match %s ? "%s" => "%s"', minion_id, tgt, results)

salt/matchers/nodegroup_match.py

@@ -10,6 +10,14 @@ import salt.utils.minions
 log = logging.getLogger(__name__)
 
 
+def _load_matchers(opts):
+    """
+    Store matchers in __context__ so they're only loaded once
+    """
+    __context__["matchers"] = {}
+    __context__["matchers"] = salt.loader.matchers(opts)
+
+
 def match(tgt, nodegroups=None, opts=None, minion_id=None):
     """
     This is a compatibility matcher and is NOT called when using
@@ -22,8 +30,9 @@ def match(tgt, nodegroups=None, opts=None, minion_id=None):
         log.debug("Nodegroup matcher called with no nodegroups.")
         return False
     if tgt in nodegroups:
-        matchers = salt.loader.matchers(opts)
-        return matchers["compound_match.match"](
+        if "matchers" not in __context__:
+            _load_matchers(opts)
+        return __context__["matchers"]["compound_match.match"](
             salt.utils.minions.nodegroup_comp(tgt, nodegroups)
         )
     return False

salt/modules/cmdmod.py

@@ -933,6 +933,7 @@ def _run_quiet(
     success_retcodes=None,
     success_stdout=None,
     success_stderr=None,
+    ignore_retcode=None,
 ):
     """
     Helper for running commands quietly for minion startup
@@ -959,6 +960,7 @@ def _run_quiet(
         success_retcodes=success_retcodes,
         success_stdout=success_stdout,
         success_stderr=success_stderr,
+        ignore_retcode=ignore_retcode,
     )["stdout"]
 
 
@@ -981,6 +983,7 @@ def _run_all_quiet(
     success_retcodes=None,
     success_stdout=None,
     success_stderr=None,
+    ignore_retcode=None,
 ):
 
     """
@@ -1013,6 +1016,7 @@ def _run_all_quiet(
         success_retcodes=success_retcodes,
         success_stdout=success_stdout,
         success_stderr=success_stderr,
+        ignore_retcode=ignore_retcode,
     )
 
 

salt/modules/dpkg_lowpkg.py

@@ -319,7 +319,7 @@ def _get_pkg_license(pkg):
     licenses = set()
     cpr = "/usr/share/doc/{}/copyright".format(pkg)
     if os.path.exists(cpr):
-        with salt.utils.files.fopen(cpr) as fp_:
+        with salt.utils.files.fopen(cpr, errors="ignore") as fp_:
             for line in salt.utils.stringutils.to_unicode(fp_.read()).split(os.linesep):
                 if line.startswith("License:"):
                     licenses.add(line.split(":", 1)[1].strip())

salt/modules/file.py

@@ -3709,7 +3709,7 @@ def is_link(path):
     return os.path.islink(os.path.expanduser(path))
 
 
-def symlink(src, path, force=False):
+def symlink(src, path, force=False, atomic=False):
     """
     Create a symbolic link (symlink, soft link) to a file
 
@@ -3723,8 +3723,12 @@ def symlink(src, path, force=False):
             Overwrite an existing symlink with the same name
             .. versionadded:: 3005
 
+        atomic (bool):
+            Use atomic file operations to create the symlink
+            .. versionadded:: 3006.0
+
     Returns:
-        bool: True if successful, otherwise False
+        bool: ``True`` if successful, otherwise raises ``CommandExecutionError``
 
     CLI Example:
 
@@ -3734,6 +3738,9 @@ def symlink(src, path, force=False):
     """
     path = os.path.expanduser(path)
 
+    if not os.path.isabs(path):
+        raise SaltInvocationError("Link path must be absolute: {}".format(path))
+
     if os.path.islink(path):
         try:
             if os.path.normpath(salt.utils.path.readlink(path)) == os.path.normpath(
@@ -3744,18 +3751,32 @@ def symlink(src, path, force=False):
         except OSError:
             pass
 
-        if force:
-            os.unlink(path)
-        else:
+        if not force and not atomic:
             msg = "Found existing symlink: {}".format(path)
             raise CommandExecutionError(msg)
 
-    if os.path.exists(path):
+    if os.path.exists(path) and not force and not atomic:
         msg = "Existing path is not a symlink: {}".format(path)
         raise CommandExecutionError(msg)
 
-    if not os.path.isabs(path):
-        raise SaltInvocationError("Link path must be absolute: {}".format(path))
+    if (os.path.islink(path) or os.path.exists(path)) and force and not atomic:
+        os.unlink(path)
+    elif atomic:
+        link_dir = os.path.dirname(path)
+        retry = 0
+        while retry < 5:
+            temp_link = tempfile.mktemp(dir=link_dir)
+            try:
+                os.symlink(src, temp_link)
+                break
+            except FileExistsError:
+                retry += 1
+        try:
+            os.replace(temp_link, path)
+            return True
+        except OSError:
+            os.remove(temp_link)
+            raise CommandExecutionError("Could not create '{}'".format(path))
 
     try:
         os.symlink(src, path)

salt/modules/freebsd_sysctl.py

@@ -167,7 +167,12 @@ def persist(name, value, config="/etc/sysctl.conf"):
                     rest_v = rest.split()[0]
                     rest = rest[len(rest_v) :]
                 if rest_v == value:
-                    return "Already set"
+                    # If it is correct in the config file, check in memory
+                    if str(get(name)) != value:
+                        assign(name, value)
+                        return "Updated"
+                    else:
+                        return "Already set"
                 new_line = _formatfor(key, value, config, rest)
                 nlines.append(new_line)
                 edited = True

salt/modules/groupadd.py

@@ -13,7 +13,9 @@ import logging
 import os
 
 import salt.utils.files
+import salt.utils.path
 import salt.utils.stringutils
+from salt.exceptions import CommandExecutionError
 
 try:
     import grp
@@ -40,6 +42,16 @@ def __virtual__():
     )
 
 
+def _which(cmd):
+    """
+    Utility function wrapper to error out early if a command is not found
+    """
+    _cmd = salt.utils.path.which(cmd)
+    if not _cmd:
+        raise CommandExecutionError("Command '{}' cannot be found".format(cmd))
+    return _cmd
+
+
 def add(name, gid=None, system=False, root=None, non_unique=False):
     """
     .. versionchanged:: 3006.0
@@ -69,7 +81,7 @@ def add(name, gid=None, system=False, root=None, non_unique=False):
 
         salt '*' group.add foo 3456
     """
-    cmd = ["groupadd"]
+    cmd = [_which("groupadd")]
     if gid:
         cmd.append("-g {}".format(gid))
         if non_unique:
@@ -103,7 +115,7 @@ def delete(name, root=None):
 
         salt '*' group.delete foo
     """
-    cmd = ["groupdel"]
+    cmd = [_which("groupdel")]
 
     if root is not None:
         cmd.extend(("-R", root))
@@ -198,7 +210,7 @@ def _chattrib(name, key, value, param, root=None):
     if value == pre_info[key]:
         return True
 
-    cmd = ["groupmod"]
+    cmd = [_which("groupmod")]
 
     if root is not None:
         cmd.extend(("-R", root))
@@ -274,15 +286,15 @@ def adduser(name, username, root=None):
 
     if __grains__["kernel"] == "Linux":
         if on_redhat_5:
-            cmd = ["gpasswd", "-a", username, name]
+            cmd = [_which("gpasswd"), "-a", username, name]
         elif on_suse_11:
-            cmd = ["usermod", "-A", name, username]
+            cmd = [_which("usermod"), "-A", name, username]
         else:
-            cmd = ["gpasswd", "--add", username, name]
+            cmd = [_which("gpasswd"), "--add", username, name]
         if root is not None:
             cmd.extend(("--root", root))
     else:
-        cmd = ["usermod", "-G", name, username]
+        cmd = [_which("usermod"), "-G", name, username]
         if root is not None:
             cmd.extend(("-R", root))
 
@@ -327,11 +339,11 @@ def deluser(name, username, root=None):
         if username in grp_info["members"]:
             if __grains__["kernel"] == "Linux":
                 if on_redhat_5:
-                    cmd = ["gpasswd", "-d", username, name]
+                    cmd = [_which("gpasswd"), "-d", username, name]
                 elif on_suse_11:
-                    cmd = ["usermod", "-R", name, username]
+                    cmd = [_which("usermod"), "-R", name, username]
                 else:
-                    cmd = ["gpasswd", "--del", username, name]
+                    cmd = [_which("gpasswd"), "--del", username, name]
                 if root is not None:
                     cmd.extend(("--root", root))
                 retcode = __salt__["cmd.retcode"](cmd, python_shell=False)
@@ -339,7 +351,7 @@ def deluser(name, username, root=None):
                 out = __salt__["cmd.run_stdout"](
                     "id -Gn {}".format(username), python_shell=False
                 )
-                cmd = ["usermod", "-S"]
+                cmd = [_which("usermod"), "-S"]
                 cmd.append(",".join([g for g in out.split() if g != str(name)]))
                 cmd.append("{}".format(username))
                 retcode = __salt__["cmd.retcode"](cmd, python_shell=False)
@@ -349,6 +361,8 @@ def deluser(name, username, root=None):
             return not retcode
         else:
             return True
+    except CommandExecutionError:
+        raise
     except Exception:  # pylint: disable=broad-except
         return True
 
@@ -386,15 +400,16 @@ def members(name, members_list, root=None):
 
     if __grains__["kernel"] == "Linux":
         if on_redhat_5:
-            cmd = ["gpasswd", "-M", members_list, name]
+            cmd = [_which("gpasswd"), "-M", members_list, name]
         elif on_suse_11:
             for old_member in __salt__["group.info"](name).get("members"):
                 __salt__["cmd.run"](
-                    "groupmod -R {} {}".format(old_member, name), python_shell=False
+                    "{} -R {} {}".format(_which("groupmod"), old_member, name),
+                    python_shell=False,
                 )
-            cmd = ["groupmod", "-A", members_list, name]
+            cmd = [_which("groupmod"), "-A", members_list, name]
         else:
-            cmd = ["gpasswd", "--members", members_list, name]
+            cmd = [_which("gpasswd"), "--members", members_list, name]
         if root is not None:
             cmd.extend(("--root", root))
         retcode = __salt__["cmd.retcode"](cmd, python_shell=False)
@@ -402,14 +417,17 @@ def members(name, members_list, root=None):
         retcode = 1
         grp_info = __salt__["group.info"](name)
         if grp_info and name in grp_info["name"]:
-            __salt__["cmd.run"]("groupdel {}".format(name), python_shell=False)
             __salt__["cmd.run"](
-                "groupadd -g {} {}".format(grp_info["gid"], name), python_shell=False
+                "{} {}".format(_which("groupdel"), name), python_shell=False
+            )
+            __salt__["cmd.run"](
+                "{} -g {} {}".format(_which("groupadd"), grp_info["gid"], name),
+                python_shell=False,
             )
             for user in members_list.split(","):
                 if user:
                     retcode = __salt__["cmd.retcode"](
-                        ["usermod", "-G", name, user], python_shell=False
+                        [_which("usermod"), "-G", name, user], python_shell=False
                     )
                     if not retcode == 0:
                         break

salt/modules/iptables.py

@@ -744,7 +744,8 @@ def check(table="filter", chain=None, rule=None, family="ipv4"):
 
     if _has_option("--check", family):
         cmd = "{} -t {} -C {} {}".format(ipt_cmd, table, chain, rule)
-        out = __salt__["cmd.run_stderr"](cmd, output_loglevel="quiet")
+        __salt__["cmd.run_stderr"](cmd, output_loglevel="quiet")
+        return not __context__["retcode"]
     else:
         _chain_name = hex(uuid.getnode())
 

salt/modules/kmod.py

@@ -8,6 +8,7 @@ import re
 
 import salt.utils.files
 import salt.utils.path
+from salt.exceptions import CommandExecutionError
 
 log = logging.getLogger(__name__)
 
@@ -108,6 +109,16 @@ def _remove_persistent_module(mod, comment):
     return {mod_name}
 
 
+def _which(cmd):
+    """
+    Utility function wrapper to error out early if a command is not found
+    """
+    _cmd = salt.utils.path.which(cmd)
+    if not _cmd:
+        raise CommandExecutionError("Command '{}' cannot be found".format(cmd))
+    return _cmd
+
+
 def available():
     """
     Return a list of all available kernel modules
@@ -169,7 +180,7 @@ def lsmod():
         salt '*' kmod.lsmod
     """
     ret = []
-    for line in __salt__["cmd.run"]("lsmod").splitlines():
+    for line in __salt__["cmd.run"](_which("lsmod")).splitlines():
         comps = line.split()
         if not len(comps) > 2:
             continue
@@ -237,7 +248,9 @@ def load(mod, persist=False):
         salt '*' kmod.load kvm
     """
     pre_mods = lsmod()
-    res = __salt__["cmd.run_all"]("modprobe {}".format(mod), python_shell=False)
+    res = __salt__["cmd.run_all"](
+        "{} {}".format(_which("modprobe"), mod), python_shell=False
+    )
     if res["retcode"] == 0:
         post_mods = lsmod()
         mods = _new_mods(pre_mods, post_mods)
@@ -283,7 +296,9 @@ def remove(mod, persist=False, comment=True):
         salt '*' kmod.remove kvm
     """
     pre_mods = lsmod()
-    res = __salt__["cmd.run_all"]("rmmod {}".format(mod), python_shell=False)
+    res = __salt__["cmd.run_all"](
+        "{} {}".format(_which("rmmod"), mod), python_shell=False
+    )
     if res["retcode"] == 0:
         post_mods = lsmod()
         mods = _rm_mods(pre_mods, post_mods)