Merge 3006.x into master

.github/workflows/ci.yml

@@ -100,6 +100,7 @@ jobs:
                 - .github/actions/**/action.yml
                 - .github/workflows/*.yml
                 - .github/workflows/templates/*.yml.jinja2
+                - tools/precommit/workflows.py
             salt:
               - added|modified: &salt_added_modified
                 - setup.py

.github/workflows/nightly.yml

@@ -144,6 +144,7 @@ jobs:
                 - .github/actions/**/action.yml
                 - .github/workflows/*.yml
                 - .github/workflows/templates/*.yml.jinja2
+                - tools/precommit/workflows.py
             salt:
               - added|modified: &salt_added_modified
                 - setup.py

.github/workflows/scheduled.yml

@@ -134,6 +134,7 @@ jobs:
                 - .github/actions/**/action.yml
                 - .github/workflows/*.yml
                 - .github/workflows/templates/*.yml.jinja2
+                - tools/precommit/workflows.py
             salt:
               - added|modified: &salt_added_modified
                 - setup.py

.github/workflows/staging.yml

@@ -130,6 +130,7 @@ jobs:
                 - .github/actions/**/action.yml
                 - .github/workflows/*.yml
                 - .github/workflows/templates/*.yml.jinja2
+                - tools/precommit/workflows.py
             salt:
               - added|modified: &salt_added_modified
                 - setup.py

.github/workflows/templates/layout.yml.jinja

@@ -148,6 +148,7 @@ jobs:
                 - .github/actions/**/action.yml
                 - .github/workflows/*.yml
                 - .github/workflows/templates/*.yml.jinja2
+                - tools/precommit/workflows.py
             salt:
               - added|modified: &salt_added_modified
                 - setup.py

.github/workflows/templates/test-package-downloads-action.yml.jinja

@@ -318,7 +318,7 @@ jobs:
       - name: Download Onedir Tarball as an Artifact
         uses: actions/download-artifact@v3
         with:
-          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-darwin-${{ matrix.arch }}.tar.xz
+          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-darwin-${{ matrix.arch == 'arm64' && 'aarch64' || matrix.arch }}.tar.xz
           path: artifacts/
 
       - name: Install System Dependencies
@@ -330,7 +330,7 @@ jobs:
         run: |
           python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
           cd artifacts
-          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-darwin-${{ matrix.arch }}.tar.xz
+          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-darwin-${{ matrix.arch == 'arm64' && 'aarch64' || matrix.arch }}.tar.xz
 
       - name: Set up Python ${{ inputs.python-version }}
         uses: actions/setup-python@v4

.github/workflows/test-package-downloads-action.yml

@@ -442,10 +442,10 @@ jobs:
             arch: x86_64
             pkg-type: package
           - distro-slug: macos-13-xlarge
-            arch: aarch64
+            arch: arm64
             pkg-type: package
           - distro-slug: macos-13-xlarge
-            arch: aarch64
+            arch: arm64
             pkg-type: onedir
 
     steps:
@@ -461,7 +461,7 @@ jobs:
       - name: Download Onedir Tarball as an Artifact
         uses: actions/download-artifact@v3
         with:
-          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-darwin-${{ matrix.arch }}.tar.xz
+          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-darwin-${{ matrix.arch == 'arm64' && 'aarch64' || matrix.arch }}.tar.xz
           path: artifacts/
 
       - name: Install System Dependencies
@@ -473,7 +473,7 @@ jobs:
         run: |
           python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
           cd artifacts
-          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-darwin-${{ matrix.arch }}.tar.xz
+          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-darwin-${{ matrix.arch == 'arm64' && 'aarch64' || matrix.arch }}.tar.xz
 
       - name: Set up Python ${{ inputs.python-version }}
         uses: actions/setup-python@v4

.pre-commit-config.yaml

@@ -59,7 +59,7 @@ repos:
       - id: tools
         alias: generate-workflows
         name: Generate GitHub Workflow Templates
-        files: ^(cicd/shared-gh-workflows-context\.yml|tools/pre_commit\.py|.github/workflows/templates/.*)$
+        files: ^(cicd/shared-gh-workflows-context\.yml|tools/precommit/workflows\.py|.github/workflows/templates/.*)$
         pass_filenames: false
         args:
           - pre-commit

SECURITY.md

@@ -1,7 +1,7 @@
 # SaltStack's Security Disclosure Policy
 
 **Email**
-- security@saltstack.com
+- saltproject-security.pdl@broadcom.com
 
 **GPG key ID:**
 - 4EA0793D
@@ -78,7 +78,7 @@ vtBIWO4LPeGEvb2Gs65PL2eouOqU6yvBr5Y=
 -----END PGP PUBLIC KEY BLOCK-----
 ```
 
-The SaltStack Security Team is available at security@saltstack.com for
+The SaltStack Security Team is available at saltproject-security.pdl@broadcom.com for
 security-related bug reports or questions.
 
 We request the disclosure of any security-related bugs or issues be reported
@@ -92,7 +92,7 @@ SaltStack takes security and the trust of our customers and users very
 seriously. Our disclosure policy is intended to resolve security issues as
 quickly and safely as is possible.
 
-1.  A security report sent to security@saltstack.com is assigned to a team
+1.  A security report sent to saltproject-security.pdl@broadcom.com is assigned to a team
     member. This person is the primary contact for questions and will
     coordinate the fix, release, and announcement.
 

changelog/64473.fixed.md

@@ -0,0 +1 @@
+Fill out lsb_distrib_xxxx (best estimate) grains if problems with retrieving lsb_release data

changelog/65450.fixed.md

@@ -0,0 +1 @@
+Fix regression in file module which was not re-using a file client.

doc/man/salt.7

@@ -23710,7 +23710,7 @@ most secure setup, only connect syndics directly to master of masters.
 .INDENT 0.0
 .TP
 .B email
-\fI\%security@saltstack.com\fP
+\fI\%saltproject-security.pdl@broadcom.com\fP
 .TP
 .B gpg key ID
 4EA0793D
@@ -23831,7 +23831,7 @@ fwPKmQ2cKnCBs5ASj1DkgUcz2c8DTUPVqg==
 .UNINDENT
 .UNINDENT
 .sp
-The SaltStack Security Team is available at \fI\%security@saltstack.com\fP for
+The SaltStack Security Team is available at \fI\%saltproject-security.pdl@broadcom.com\fP for
 security\-related bug reports or questions.
 .sp
 We request the disclosure of any security\-related bugs or issues be reported
@@ -23845,7 +23845,7 @@ seriously. Our disclosure policy is intended to resolve security issues as
 quickly and safely as is possible.
 .INDENT 0.0
 .IP 1. 3
-A security report sent to \fI\%security@saltstack.com\fP is assigned to a team
+A security report sent to \fI\%saltproject-security.pdl@broadcom.com\fP is assigned to a team
 member. This person is the primary contact for questions and will
 coordinate the fix, release, and announcement.
 .IP 2. 3

doc/security/index.rst

@@ -4,7 +4,7 @@
 Security disclosure policy
 ==========================
 
-:email: security@saltstack.com
+:email: saltproject-security.pdl@broadcom.com
 :gpg key ID: 4EA0793D
 :gpg key fingerprint: ``8ABE 4EFC F0F4 B24B FF2A  AF90 D570 F2D3 4EA0 793D``
 
@@ -114,7 +114,7 @@ Security disclosure policy
     =i1Tf
     -----END PGP PUBLIC KEY BLOCK-----
 
-The SaltStack Security Team is available at security@saltstack.com for
+The SaltStack Security Team is available at saltproject-security.pdl@broadcom.com for
 security-related bug reports or questions.
 
 We request the disclosure of any security-related bugs or issues be reported
@@ -129,7 +129,7 @@ SaltStack takes security and the trust of our customers and users very
 seriously. Our disclosure policy is intended to resolve security issues as
 quickly and safely as is possible.
 
-1.  A security report sent to security@saltstack.com is assigned to a team
+1.  A security report sent to saltproject-security.pdl@broadcom.com is assigned to a team
     member. This person is the primary contact for questions and will
     coordinate the fix, release, and announcement.
 

doc/topics/development/modules/developing.rst

@@ -155,7 +155,7 @@ The following dunder dictionaries are always defined, but may be empty
 __opts__
 --------
 
-..versionchanged:: 3006.0
+.. versionchanged:: 3006.0
 
     The ``__opts__`` dictionary can now be accessed via
     :py:mod:`~salt.loader.context``.
@@ -248,13 +248,6 @@ executions until the modules are refreshed; such as when
 :py:func:`saltutil.sync_all <salt.modules.saltutil.sync_all>` or
 :py:func:`state.apply <salt.modules.state.apply_>` are executed.
 
-A great place to see how to use ``__context__`` is in the cp.py module in
-salt/modules/cp.py. The fileclient authenticates with the master when it is
-instantiated and then is used to copy files to the minion. Rather than create a
-new fileclient for each file that is to be copied down, one instance of the
-fileclient is instantiated in the ``__context__`` dictionary and is reused for
-each file. Here is an example from salt/modules/cp.py:
-
 .. code-block:: python
 
     if not "cp.fileclient" in __context__:
@@ -303,3 +296,13 @@ Defined in: State
 __sdb__
 -------
 Defined in: SDB
+
+
+__file_client__
+---------------
+
+.. versionchanged:: 3006.5
+
+The ``__file_client__`` dunder was added to states and execution modules. This
+enables the use of a file client without haveing to instantiate one in
+the module.

pkg/rpm/salt.spec

@@ -50,6 +50,9 @@ Requires: dmidecode
 Requires: pciutils
 Requires: which
 Requires: openssl
+Requires: /usr/sbin/usermod
+Requires: /usr/sbin/groupadd
+Requires: /usr/sbin/useradd
 
 BuildRequires: python3
 BuildRequires: python3-pip

pkg/tests/download/test_pkg_download.py

@@ -401,8 +401,6 @@ def setup_macos(
 ):
     arch = os.environ.get("SALT_REPO_ARCH") or "x86_64"
     if package_type == "package":
-        if arch == "aarch64":
-            arch = "arm64"
 
         if packaging.version.parse(salt_release) > packaging.version.parse("3005"):
             mac_pkg = f"salt-{salt_release}-py3-{arch}.pkg"

salt/fileclient.py

@@ -1525,3 +1525,17 @@ class DumbAuth:
 
     def gen_token(self, clear_tok):
         return clear_tok
+
+
+class ContextlessFileClient:
+    def __init__(self, file_client):
+        self.file_client = file_client
+
+    def __getattr__(self, key):
+        return getattr(self.file_client, key)
+
+    def __exit__(self, *_):
+        pass
+
+    def __enter__(self):
+        return self

salt/grains/core.py

@@ -2097,9 +2097,11 @@ def _os_release_quirks_for_osrelease(os_release):
     if os_release["ID"] in ("mendel",):
         # Mendel sets VERSION_CODENAME but not VERSION_ID.
         # Only PRETTY_NAME mentions the version number.
-        match = _PRETTY_NAME_RE.match(os_release["PRETTY_NAME"])
-        if match:
-            return match.group("version")
+        # for example: Mendel GNU/Linux 5 (Eagle)
+        test_strg = os_release["PRETTY_NAME"].split()
+        if len(test_strg) >= 3:
+            return test_strg[2]
+
     return None
 
 
@@ -2166,8 +2168,9 @@ def _linux_distribution_data():
             "rocky",
             "ubuntu",
         ):
-            # Solely use os-release data. See description of the function.
-            return grains
+            if lsb_has_error is False:
+                # Solely use os-release data. See description of the function.
+                return grains
 
     except OSError:
         os_release = {}
@@ -2221,6 +2224,10 @@ def _legacy_linux_distribution_data(grains, os_release, lsb_has_error):
                             cpe.get("version") and cpe.get("vendor") == "opensuse"
                         ):  # Keep VERSION_ID for SLES
                             grains["lsb_distrib_release"] = cpe["version"]
+                if "ID" in os_release and os_release["ID"].strip() == "mendel":
+                    test_strg = os_release["PRETTY_NAME"].split()
+                    if len(test_strg) >= 3:
+                        grains["lsb_distrib_release"] = test_strg[2]
 
             elif os.path.isfile("/etc/SuSE-release"):
                 log.trace("Parsing distrib info from /etc/SuSE-release")
@@ -2338,6 +2345,20 @@ def _legacy_linux_distribution_data(grains, os_release, lsb_has_error):
         ):
             grains.pop("lsb_distrib_release", None)
         grains["osrelease"] = grains.get("lsb_distrib_release", osrelease).strip()
+
+    # allow for codename being within brackets on certain OS
+    if grains.get("lsb_distrib_codename", "") and (
+        any(os in grains.get("os", "") for os in ["Rocky", "AlmaLinux", "AstraLinuxSE"])
+    ):
+        test_strg = grains["lsb_distrib_codename"].split("(", maxsplit=1)
+        if len(test_strg) >= 2:
+            test_strg_2 = test_strg[1].split(")", maxsplit=1)
+            if grains["os"] == "AstraLinuxSE":
+                # AstraLinuxSE has version aka 'Smolensk 1.6'
+                grains["lsb_distrib_codename"] = test_strg_2[0].split()[0].lower()
+            else:
+                grains["lsb_distrib_codename"] = test_strg_2[0]
+
     grains["oscodename"] = grains.get("lsb_distrib_codename", "").strip() or oscodename
     if "Red Hat" in grains["oscodename"]:
         grains["oscodename"] = oscodename

salt/loader/__init__.py

@@ -263,6 +263,7 @@ def minion_mods(
     notify=False,
     static_modules=None,
     proxy=None,
+    file_client=None,
 ):
     """
     Load execution modules
@@ -314,6 +315,7 @@ def minion_mods(
             "__utils__": utils,
             "__proxy__": proxy,
             "__opts__": opts,
+            "__file_client__": file_client,
         },
         whitelist=whitelist,
         loaded_base_name=loaded_base_name,
@@ -778,6 +780,7 @@ def states(
     proxy=None,
     context=None,
     loaded_base_name=None,
+    file_client=None,
 ):
     """
     Returns the state modules
@@ -815,6 +818,7 @@ def states(
             "__utils__": utils,
             "__serializers__": serializers,
             "__context__": context,
+            "__file_client__": file_client,
         },
         whitelist=whitelist,
         extra_module_dirs=utils.module_dirs if utils else None,

salt/loader/context.py

@@ -84,11 +84,7 @@ class NamedLoaderContext(collections.abc.MutableMapping):
         self.value()[item] = value
 
     def __bool__(self):
-        try:
-            self.loader
-        except LookupError:
-            return False
-        return True
+        return bool(self.value())
 
     def __len__(self):
         return self.value().__len__()

salt/loader/dunder.py

@@ -0,0 +1,9 @@
+"""
+Salt dunders.
+"""
+import salt.loader.context
+
+loader_context = salt.loader.context.LoaderContext()
+
+
+__file_client__ = loader_context.named_context("__file_client__")

salt/modules/cp.py

@@ -20,6 +20,7 @@ import salt.utils.path
 import salt.utils.templates
 import salt.utils.url
 from salt.exceptions import CommandExecutionError
+from salt.loader.dunder import __file_client__
 
 log = logging.getLogger(__name__)
 
@@ -159,8 +160,13 @@ def recv_chunked(dest, chunk, append=False, compressed=True, mode=None):
 
 def _client():
     """
-    Return a client, hashed by the list of masters
+    Return a file client
+
+    If the __file_client__ context is set return it, otherwize create a new
+    file client using __opts__.
     """
+    if __file_client__:
+        return __file_client__.value()
     return salt.fileclient.get_file_client(__opts__)
 
 

salt/modules/dockermod.py

@@ -222,6 +222,7 @@ import salt.utils.functools
 import salt.utils.json
 import salt.utils.path
 from salt.exceptions import CommandExecutionError, SaltInvocationError
+from salt.loader.dunder import __file_client__
 from salt.state import HighState
 
 __docformat__ = "restructuredtext en"
@@ -331,6 +332,18 @@ def __virtual__():
     return (False, "Could not import docker module, is docker-py installed?")
 
 
+def _file_client():
+    """
+    Return a file client
+
+    If the __file_client__ context is set return it, otherwize create a new
+    file client using __opts__.
+    """
+    if __file_client__:
+        return __file_client__.value()
+    return salt.fileclient.get_file_client(__opts__)
+
+
 class DockerJSONDecoder(json.JSONDecoder):
     def decode(self, s, _w=None):
         objs = []
@@ -6639,7 +6652,7 @@ def _prepare_trans_tar(name, sls_opts, mods=None, pillar=None, extra_filerefs=""
     # reuse it from salt.ssh, however this function should
     # be somewhere else
     refs = salt.client.ssh.state.lowstate_file_refs(chunks, extra_filerefs)
-    with salt.fileclient.get_file_client(__opts__) as fileclient:
+    with _file_client() as fileclient:
         return salt.client.ssh.state.prep_trans_tar(
             fileclient, chunks, refs, pillar, name
         )

salt/state.py

@@ -759,6 +759,7 @@ class State:
         mocked=False,
         loader="states",
         initial_pillar=None,
+        file_client=None,
     ):
         self._init_kwargs = {
             "opts": opts,
@@ -775,6 +776,12 @@ class State:
         if "grains" not in opts:
             opts["grains"] = salt.loader.grains(opts)
         self.opts = opts
+        if file_client:
+            self.file_client = file_client
+            self.preserve_file_client = True
+        else:
+            self.file_client = salt.fileclient.get_file_client(self.opts)
+            self.preserve_file_client = False
         self.proxy = proxy
         self._pillar_override = pillar_override
         if pillar_enc is not None:
@@ -799,7 +806,11 @@ class State:
                     self.opts.get("pillar_merge_lists", False),
                 )
         log.debug("Finished gathering pillar data for state run")
-        self.state_con = context or {}
+        if context is None:
+            self.state_con = {}
+        else:
+            self.state_con = context
+        self.state_con["fileclient"] = self.file_client
         self.load_modules()
         self.active = set()
         self.mod_init = set()
@@ -1277,6 +1288,7 @@ class State:
                 self.serializers,
                 context=self.state_con,
                 proxy=self.proxy,
+                file_client=salt.fileclient.ContextlessFileClient(self.file_client),
             )
 
     def load_modules(self, data=None, proxy=None):
@@ -1286,7 +1298,11 @@ class State:
         log.info("Loading fresh modules for state activity")
         self.utils = salt.loader.utils(self.opts)
         self.functions = salt.loader.minion_mods(
-            self.opts, self.state_con, utils=self.utils, proxy=self.proxy
+            self.opts,
+            self.state_con,
+            utils=self.utils,
+            proxy=self.proxy,
+            file_client=salt.fileclient.ContextlessFileClient(self.file_client),
         )
         if isinstance(data, dict):
             if data.get("provider", False):
@@ -3671,6 +3687,16 @@ class State:
             return errors
         return self.call_high(high)
 
+    def destroy(self):
+        if not self.preserve_file_client:
+            self.file_client.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_):
+        self.destroy()
+
 
 class LazyAvailStates:
     """
@@ -4949,9 +4975,15 @@ class HighState(BaseHighState):
         mocked=False,
         loader="states",
         initial_pillar=None,
+        file_client=None,
     ):
         self.opts = opts
-        self.client = salt.fileclient.get_file_client(self.opts)
+        if file_client:
+            self.client = file_client
+            self.preserve_client = True
+        else:
+            self.client = salt.fileclient.get_file_client(self.opts)
+            self.preserve_client = False
         BaseHighState.__init__(self, opts)
         self.state = State(
             self.opts,
@@ -4963,6 +4995,7 @@ class HighState(BaseHighState):
             mocked=mocked,
             loader=loader,
             initial_pillar=initial_pillar,
+            file_client=self.client,
         )
         self.matchers = salt.loader.matchers(self.opts)
         self.proxy = proxy
@@ -4997,7 +5030,8 @@ class HighState(BaseHighState):
             return None
 
     def destroy(self):
-        self.client.destroy()
+        if not self.preserve_client:
+            self.client.destroy()
 
     def __enter__(self):
         return self

salt/states/ansiblegate.py

@@ -38,12 +38,25 @@ import sys
 
 import salt.fileclient
 import salt.utils.decorators.path
+from salt.loader.dunder import __file_client__
 from salt.utils.decorators import depends
 
 log = logging.getLogger(__name__)
 __virtualname__ = "ansible"
 
 
+def _file_client():
+    """
+    Return a file client
+
+    If the __file_client__ context is set return it, otherwize create a new
+    file client using __opts__.
+    """
+    if __file_client__:
+        return __file_client__.value()
+    return salt.fileclient.get_file_client(__opts__)
+
+
 @depends("ansible")
 class AnsibleState:
     """
@@ -162,7 +175,7 @@ def playbooks(name, rundir=None, git_repo=None, git_kwargs=None, ansible_kwargs=
     }
     if git_repo:
         if not isinstance(rundir, str) or not os.path.isdir(rundir):
-            with salt.fileclient.get_file_client(__opts__) as client:
+            with _file_client() as client:
                 rundir = client._extrn_path(git_repo, "base")
             log.trace("rundir set to %s", rundir)
         if not isinstance(git_kwargs, dict):

tests/pytests/functional/grains/test_grains.py

@@ -0,0 +1,23 @@
+import pytest
+
+pytestmark = [
+    pytest.mark.skip_unless_on_linux,
+    pytest.mark.skipif(
+        'grains["os_family"] != "Debian"',
+        reason="Tests applicable only to Debian and Ubuntu",
+    ),
+]
+
+
+def test_grains(grains):
+    """
+    Test to ensure that the lsb_distrib_xxxx grains are
+    populated on Debian machines
+    """
+    assert "lsb_distrib_id" in grains
+    assert "lsb_distrib_release" in grains
+    assert "lsb_distrib_codename" in grains
+
+    assert grains["lsb_distrib_id"] == grains["osfullname"]
+    assert grains["lsb_distrib_release"] == grains["osrelease"]
+    assert grains["lsb_distrib_codename"] == grains["oscodename"]

tests/pytests/unit/conftest.py

@@ -16,6 +16,9 @@ def minion_opts(tmp_path):
     opts = salt.config.DEFAULT_MINION_OPTS.copy()
     opts["__role"] = "minion"
     opts["root_dir"] = str(root_dir)
+    opts["master_uri"] = "tcp://{ip}:{port}".format(
+        ip="127.0.0.1", port=opts["master_port"]
+    )
     for name in ("cachedir", "pki_dir", "sock_dir", "conf_dir"):
         dirpath = root_dir / name
         dirpath.mkdir(parents=True)

tools/ci.py

@@ -1272,8 +1272,22 @@ def upload_coverage(ctx: Context, reports_path: pathlib.Path, commit_sha: str =
                 "--flags",
                 flags,
                 check=False,
+                capture=True,
             )
+            stdout = ret.stdout.strip().decode()
+            stderr = ret.stderr.strip().decode()
             if ret.returncode == 0:
+                ctx.console_stdout.print(stdout)
+                ctx.console.print(stderr)
+                break
+
+            if (
+                "Too many uploads to this commit" in stdout
+                or "Too many uploads to this commit" in stderr
+            ):
+                # Let's just stop trying
+                ctx.console_stdout.print(stdout)
+                ctx.console.print(stderr)
                 break
 
             if current_attempt >= max_attempts:

tools/precommit/workflows.py

@@ -266,8 +266,12 @@ def generate_workflows(ctx: Context):
             continue
         test_salt_pkg_downloads_listing["linux"].append((slug, arch, "onedir"))
     for slug, display_name, arch in build_ci_deps_listing["macos"]:
+        if arch == "aarch64":
+            arch = "arm64"
         test_salt_pkg_downloads_listing["macos"].append((slug, arch, "package"))
     for slug, display_name, arch in build_ci_deps_listing["macos"][-1:]:
+        if arch == "aarch64":
+            arch = "arm64"
         test_salt_pkg_downloads_listing["macos"].append((slug, arch, "onedir"))
     for slug, display_name, arch in build_ci_deps_listing["windows"][-1:]:
         for pkg_type in ("nsis", "msi", "onedir"):