Distro is now a hard dependency

Refs #55310
Refs #55835

.pylintrc

@@ -684,7 +684,8 @@ allowed-3rd-party-modules=msgpack,
                           pytest,
                           setuptools,
                           pytestsalt,
-                          saltfactories
+                          saltfactories,
+                          distro
 
 
 [EXCEPTIONS]

doc/conf.py

@@ -94,6 +94,7 @@ MOCK_MODULES = [
     "Crypto.Random",
     "Crypto.Signature",
     "Crypto.Signature.PKCS1_v1_5",
+    "distro",
     "M2Crypto",
     "msgpack",
     "yaml",

pkg/osx/req.txt

@@ -5,6 +5,7 @@ cffi==1.12.2
 CherryPy==17.4.1
 click==7.0
 cryptography==2.6.1
+distro==1.5.0
 gitpython==2.1.15
 idna==2.8
 ipaddress==1.0.22

pkg/windows/req.txt

@@ -4,7 +4,7 @@ certifi
 cffi==1.12.2
 CherryPy==17.4.1
 cryptography==2.6.1
-distro==1.4.0
+distro==1.5.0
 idna==2.8
 ioloop==0.1a0
 ipaddress==1.0.22
@@ -19,7 +19,7 @@ pyasn1==0.4.5
 pycparser==2.19
 pycryptodomex==3.9.7
 pycurl==7.43.0.5
-pymssql==2.1.4
+pymssql==2.1.4 ; python_version < "3.8"
 PyMySQL==0.9.3
 pyopenssl==19.0.0
 python-dateutil==2.8.0

requirements/base.txt

@@ -3,6 +3,4 @@ msgpack>=0.5,!=0.5.5,<1.0.0
 PyYAML
 MarkupSafe
 requests>=1.0.0
-# Requirements for Tornado 4.5.3 (vendored as salt.ext.tornado)
+distro
-singledispatch==3.4.0.3; python_version < '3.4'
-# Required by Tornado to handle threads stuff.

requirements/static/py3.5/darwin.txt

@@ -31,6 +31,7 @@ contextlib2==0.5.5        # via cherrypy
 croniter==0.3.29
 cryptography==2.6.1
 distlib==0.3.0            # via virtualenv
+distro==1.5.0
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
 docker==3.7.2

requirements/static/py3.5/linux.txt

@@ -26,6 +26,7 @@ cherrypy==17.3.0
 contextlib2==0.5.5        # via cherrypy
 croniter==0.3.29
 cryptography==2.6.1       # via moto, paramiko, pyopenssl, vcert
+distro==1.5.0
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
 docker==3.7.2

requirements/static/py3.5/windows.txt

@@ -24,7 +24,7 @@ cherrypy==17.4.1
 colorama==0.4.1           # via pytest
 contextlib2==0.5.5        # via cherrypy
 cryptography==2.6.1
-distro==1.4.0
+distro==1.5.0
 dmidecode==0.9.0
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
@@ -73,7 +73,7 @@ pycryptodome==3.8.1       # via python-jose
 pycryptodomex==3.9.7
 pycurl==7.43.0.5
 pygit2==0.28.2 ; python_version < "3.8"
-pymssql==2.1.4
+pymssql==2.1.4 ; python_version < "3.8"
 pymysql==0.9.3
 pyopenssl==19.0.0
 pyparsing==2.4.5          # via packaging

requirements/static/py3.6/darwin.txt

@@ -31,6 +31,7 @@ contextlib2==0.5.5        # via cherrypy
 croniter==0.3.29
 cryptography==2.6.1
 distlib==0.3.0            # via virtualenv
+distro==1.5.0
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
 docker==3.7.2

requirements/static/py3.6/linux.txt

@@ -26,6 +26,7 @@ cherrypy==17.3.0
 contextlib2==0.5.5        # via cherrypy
 croniter==0.3.29
 cryptography==2.6.1       # via moto, paramiko, pyopenssl, vcert
+distro==1.5.0
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
 docker==3.7.2

requirements/static/py3.6/windows.txt

@@ -24,7 +24,7 @@ cherrypy==17.4.1
 colorama==0.4.1           # via pytest
 contextlib2==0.5.5        # via cherrypy
 cryptography==2.6.1
-distro==1.4.0
+distro==1.5.0
 dmidecode==0.9.0
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
@@ -72,7 +72,7 @@ pycryptodome==3.8.1       # via python-jose
 pycryptodomex==3.9.7
 pycurl==7.43.0.5
 pygit2==0.28.2 ; python_version < "3.8"
-pymssql==2.1.4
+pymssql==2.1.4 ; python_version < "3.8"
 pymysql==0.9.3
 pyopenssl==19.0.0
 pyparsing==2.4.5          # via packaging

requirements/static/py3.7/darwin.txt

@@ -31,6 +31,7 @@ contextlib2==0.5.5        # via cherrypy
 croniter==0.3.29
 cryptography==2.6.1
 distlib==0.3.0            # via virtualenv
+distro==1.5.0
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
 docker==3.7.2

requirements/static/py3.7/linux.txt

@@ -26,6 +26,7 @@ cherrypy==17.3.0
 contextlib2==0.5.5        # via cherrypy
 croniter==0.3.29
 cryptography==2.6.1       # via moto, paramiko, pyopenssl, vcert
+distro==1.5.0
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
 docker==3.7.2

requirements/static/py3.7/windows.txt

@@ -23,7 +23,7 @@ cherrypy==17.4.1
 colorama==0.4.1           # via pytest
 contextlib2==0.5.5        # via cherrypy
 cryptography==2.6.1
-distro==1.4.0
+distro==1.5.0
 dmidecode==0.9.0
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
@@ -71,7 +71,7 @@ pycryptodome==3.8.1       # via python-jose
 pycryptodomex==3.9.7
 pycurl==7.43.0.5
 pygit2==0.28.2 ; python_version < "3.8"
-pymssql==2.1.4
+pymssql==2.1.4 ; python_version < "3.8"
 pymysql==0.9.3
 pyopenssl==19.0.0
 pyparsing==2.4.5          # via packaging

requirements/static/py3.8/darwin.txt

@@ -31,6 +31,7 @@ contextlib2==0.5.5        # via cherrypy
 croniter==0.3.29
 cryptography==2.6.1
 distlib==0.3.0            # via virtualenv
+distro==1.5.0
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
 docker==3.7.2

requirements/static/py3.8/linux.txt

@@ -27,6 +27,7 @@ cherrypy==17.3.0
 contextlib2==0.5.5        # via cherrypy
 croniter==0.3.29
 cryptography==2.6.1       # via moto, paramiko, pyopenssl, vcert
+distro==1.5.0
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
 docker==3.7.2

requirements/static/py3.9/darwin.txt

@@ -31,6 +31,7 @@ contextlib2==0.5.5        # via cherrypy
 croniter==0.3.29
 cryptography==2.6.1
 distlib==0.3.0            # via virtualenv
+distro==1.5.0
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
 docker==3.7.2

requirements/static/py3.9/linux.txt

@@ -27,6 +27,7 @@ cherrypy==17.3.0
 contextlib2==0.5.5        # via cherrypy
 croniter==0.3.29
 cryptography==2.6.1       # via moto, paramiko, pyopenssl, vcert
+distro==1.5.0
 dnspython==1.16.0
 docker-pycreds==0.4.0     # via docker
 docker==3.7.2

salt/grains/core.py

@@ -9,8 +9,6 @@ will always be executed first, so that any grains loaded here in the core
 module can be overwritten just by returning dict keys with the same value
 as those returned here
 """
-
-# Import python libs
 from __future__ import absolute_import, print_function, unicode_literals
 
 import datetime
@@ -23,10 +21,8 @@ import socket
 import sys
 import time
 import uuid
-import warnings
 from errno import EACCES, EPERM
 
-# Import salt libs
 import salt.exceptions
 import salt.log
 
@@ -42,69 +38,23 @@ import salt.utils.path
 import salt.utils.pkg.rpm
 import salt.utils.platform
 import salt.utils.stringutils
+from distro import linux_distribution
 from salt.ext import six
 from salt.ext.six.moves import range
 
-# pylint: disable=import-error
 try:
-    import dateutil.tz
+    import dateutil.tz  # pylint: disable=import-error
 
     _DATEUTIL_TZ = True
 except ImportError:
     _DATEUTIL_TZ = False
 
-__proxyenabled__ = ["*"]
-__FQDN__ = None
-
-# linux_distribution deprecated in py3.7
-try:
-    from platform import linux_distribution as _deprecated_linux_distribution
-
-    # Extend the default list of supported distros. This will be used for the
-    # /etc/DISTRO-release checking that is part of linux_distribution()
-    from platform import _supported_dists
-
-    _supported_dists += (
-        "arch",
-        "mageia",
-        "meego",
-        "vmware",
-        "bluewhite64",
-        "slamd64",
-        "ovs",
-        "system",
-        "mint",
-        "oracle",
-        "void",
-    )
-
-    def linux_distribution(**kwargs):
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
-            return _deprecated_linux_distribution(
-                supported_dists=_supported_dists, **kwargs
-            )
-
-
-except ImportError:
-    from distro import linux_distribution
-
-
-if salt.utils.platform.is_windows():
-    import salt.utils.win_osinfo
-
-
-__salt__ = {
-    "cmd.run": salt.modules.cmdmod._run_quiet,
-    "cmd.retcode": salt.modules.cmdmod._retcode_quiet,
-    "cmd.run_all": salt.modules.cmdmod._run_all_quiet,
-    "smbios.records": salt.modules.smbios.records,
-    "smbios.get": salt.modules.smbios.get,
-}
 log = logging.getLogger(__name__)
 
 HAS_WMI = False
 if salt.utils.platform.is_windows():
+    import salt.utils.win_osinfo
+
     # attempt to import the python wmi module
     # the Windows minion uses WMI for some of its grains
     try:
@@ -119,9 +69,19 @@ if salt.utils.platform.is_windows():
             "Unable to import Python wmi module, some core grains " "will be missing"
         )
 
-HAS_UNAME = True
+
-if not hasattr(os, "uname"):
+__proxyenabled__ = ["*"]
-    HAS_UNAME = False
+__FQDN__ = None
+
+__salt__ = {
+    "cmd.run": salt.modules.cmdmod._run_quiet,
+    "cmd.retcode": salt.modules.cmdmod._retcode_quiet,
+    "cmd.run_all": salt.modules.cmdmod._run_all_quiet,
+    "smbios.records": salt.modules.smbios.records,
+    "smbios.get": salt.modules.smbios.get,
+}
+
+HAS_UNAME = hasattr(os, "uname")
 
 _INTERFACES = {}
 

salt/utils/platform.py

@@ -2,30 +2,15 @@
 """
 Functions for identifying which platform a machine is
 """
-# Import Python libs
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
 import subprocess
 import sys
-import warnings
 
-# Import Salt libs
+from distro import linux_distribution
 from salt.utils.decorators import memoize as real_memoize
 
-# linux_distribution deprecated in py3.7
-try:
-    from platform import linux_distribution as _deprecated_linux_distribution
-
-    def linux_distribution(**kwargs):
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
-            return _deprecated_linux_distribution(**kwargs)
-
-
-except ImportError:
-    from distro import linux_distribution
-
 
 @real_memoize
 def is_windows():

salt/utils/thin.py

@@ -2,8 +2,6 @@
 """
 Generate the salt thin tarball from the installed python files
 """
-
-# Import python libs
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
@@ -16,11 +14,9 @@ import tarfile
 import tempfile
 import zipfile
 
-# Import third party libs
+import distro
 import jinja2
 import msgpack
-
-# Import salt libs
 import salt
 import salt.exceptions
 import salt.ext.six as _six
@@ -195,6 +191,7 @@ def get_tops_python(py_ver, exclude=None):
     """
     files = {}
     for mod in [
+        "distro",
         "jinja2",
         "yaml",
         "tornado",
@@ -245,7 +242,7 @@ def get_ext_tops(config):
     """
     config = copy.deepcopy(config)
     alternatives = {}
-    required = ["jinja2", "yaml", "tornado", "msgpack"]
+    required = ["jinja2", "yaml", "tornado", "msgpack", "distro"]
     tops = []
     for ns, cfg in salt.ext.six.iteritems(config or {}):
         alternatives[ns] = cfg
@@ -336,6 +333,7 @@ def get_tops(extra_mods="", so_mods=""):
     tops = []
     for mod in [
         salt,
+        distro,
         jinja2,
         yaml,
         tornado,

salt/version.py

@@ -2,43 +2,16 @@
 """
 Set up the version of Salt
 """
-
-# Import python libs
 from __future__ import absolute_import, print_function, unicode_literals
 
 import platform
 import re
 import sys
-import warnings
 
-# pylint: disable=invalid-name,redefined-builtin
+from distro import linux_distribution
-# Import 3rd-party libs
-from salt.ext import six
-from salt.ext.six.moves import map
 
-# linux_distribution deprecated in py3.7
+MAX_SIZE = sys.maxsize
-try:
-    from platform import linux_distribution as _deprecated_linux_distribution
-
-    def linux_distribution(**kwargs):
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
-            return _deprecated_linux_distribution(**kwargs)
-
-
-except ImportError:
-    from distro import linux_distribution
-
-
-# Don't rely on external packages in this module since it's used at install time
-if sys.version_info[0] == 3:
-    MAX_SIZE = sys.maxsize
-    string_types = (str,)
-else:
-    MAX_SIZE = sys.maxint
-    string_types = (six.string_types,)
 VERSION_LIMIT = MAX_SIZE - 200
-# pylint: enable=invalid-name,redefined-builtin
 
 # ----- ATTENTION --------------------------------------------------------------------------------------------------->
 #
@@ -94,8 +67,6 @@ class SaltStackVersion(object):
     )
     git_sha_regex = r"^" + git_sha_regex
 
-    if six.PY2:
-        git_sha_regex = git_sha_regex.decode(__salt_system_encoding__)
     git_sha_regex = re.compile(git_sha_regex)
 
     # Salt versions after 0.17.0 will be numbered like:
@@ -242,10 +213,10 @@ class SaltStackVersion(object):
         sha=None,
     ):
 
-        if isinstance(major, string_types):
+        if isinstance(major, str):
             major = int(major)
 
-        if isinstance(minor, string_types):
+        if isinstance(minor, str):
             if not minor:
                 # Empty string
                 minor = None
@@ -254,7 +225,7 @@ class SaltStackVersion(object):
 
         if bugfix is None and not self.new_version(major=major):
             bugfix = 0
-        elif isinstance(bugfix, string_types):
+        elif isinstance(bugfix, str):
             if not bugfix:
                 bugfix = None
             else:
@@ -262,21 +233,21 @@ class SaltStackVersion(object):
 
         if mbugfix is None:
             mbugfix = 0
-        elif isinstance(mbugfix, string_types):
+        elif isinstance(mbugfix, str):
             mbugfix = int(mbugfix)
 
         if pre_type is None:
             pre_type = ""
         if pre_num is None:
             pre_num = 0
-        elif isinstance(pre_num, string_types):
+        elif isinstance(pre_num, str):
             pre_num = int(pre_num)
 
         if noc is None:
             noc = 0
-        elif isinstance(noc, string_types) and noc == "n/a":
+        elif isinstance(noc, str) and noc == "n/a":
             noc = -1
-        elif isinstance(noc, string_types):
+        elif isinstance(noc, str):
             noc = int(noc)
 
         self.major = major
@@ -451,7 +422,7 @@ class SaltStackVersion(object):
 
     def __compare__(self, other, method):
         if not isinstance(other, SaltStackVersion):
-            if isinstance(other, string_types):
+            if isinstance(other, str):
                 other = SaltStackVersion.parse(other)
             elif isinstance(other, (list, tuple)):
                 other = SaltStackVersion(*other)
@@ -585,11 +556,8 @@ def __discover_version(saltstack_version):
                 **kwargs
             )
             out, err = process.communicate()
-        if six.PY3:
+        out = out.decode().strip()
-            out = out.decode()
+        err = err.decode().strip()
-            err = err.decode()
-        out = out.strip()
-        err = err.strip()
 
         if not out or err:
             return saltstack_version

tests/unit/utils/test_thin.py

@@ -57,6 +57,7 @@ class SSHThinTestCase(TestCase):
         ]
         lib_root = os.path.join(RUNTIME_VARS.TMP, "fake-libs")
         self.fake_libs = {
+            "distro": os.path.join(lib_root, "distro"),
             "jinja2": os.path.join(lib_root, "jinja2"),
             "yaml": os.path.join(lib_root, "yaml"),
             "tornado": os.path.join(lib_root, "tornado"),
@@ -64,6 +65,9 @@ class SSHThinTestCase(TestCase):
         }
 
         self.exp_ret = {
+            "distro": os.path.normpath(
+                os.path.join(RUNTIME_VARS.CODE_DIR, "distro.py")
+            ),
             "jinja2": os.path.normpath(os.path.join(RUNTIME_VARS.CODE_DIR, "jinja2")),
             "yaml": os.path.normpath(os.path.join(RUNTIME_VARS.CODE_DIR, "yaml")),
             "tornado": os.path.normpath(os.path.join(RUNTIME_VARS.CODE_DIR, "tornado")),
@@ -79,6 +83,15 @@ class SSHThinTestCase(TestCase):
         for lib, fp in self.fake_libs.items():
             if os.path.exists(fp):
                 shutil.rmtree(fp)
+        self.exc_libs = None
+        self.jinja_fp = None
+        self.ext_conf = None
+        self.tops = None
+        self.tar = None
+        self.digest = None
+        self.exp_files = None
+        self.fake_libs = None
+        self.exp_ret = None
 
     def _popen(self, return_value=None, side_effect=None, returncode=0):
         """
@@ -271,6 +284,7 @@ class SSHThinTestCase(TestCase):
                     "yaml": "/yaml/",
                     "tornado": "/tornado/tornado.py",
                     "msgpack": "msgpack.py",
+                    "distro": "distro.py",
                 },
             }
         }
@@ -278,7 +292,13 @@ class SSHThinTestCase(TestCase):
         assert out["namespace"]["py-version"] == cfg["namespace"]["py-version"]
         assert out["namespace"]["path"] == cfg["namespace"]["path"]
         assert sorted(out["namespace"]["dependencies"]) == sorted(
-            ["/tornado/tornado.py", "/jinja/foo.py", "/yaml/", "msgpack.py"]
+            [
+                "/tornado/tornado.py",
+                "/jinja/foo.py",
+                "/yaml/",
+                "msgpack.py",
+                "distro.py",
+            ]
         )
 
     @patch("salt.utils.thin.sys.argv", [None, '{"foo": "bar"}'])
@@ -355,6 +375,10 @@ class SSHThinTestCase(TestCase):
         with pytest.raises(salt.exceptions.SaltSystemExit):
             thin._get_ext_namespaces({"ns": {}})
 
+    @patch(
+        "salt.utils.thin.distro",
+        type("distro", (), {"__file__": "/site-packages/distro"}),
+    )
     @patch(
         "salt.utils.thin.salt",
         type(str("salt"), (), {"__file__": "/site-packages/salt"}),
@@ -410,6 +434,7 @@ class SSHThinTestCase(TestCase):
         :return:
         """
         base_tops = [
+            "/site-packages/distro",
             "/site-packages/salt",
             "/site-packages/jinja2",
             "/site-packages/yaml",
@@ -428,6 +453,10 @@ class SSHThinTestCase(TestCase):
         assert len(tops) == len(base_tops)
         assert sorted(tops) == sorted(base_tops)
 
+    @patch(
+        "salt.utils.thin.distro",
+        type("distro", (), {"__file__": "/site-packages/distro"}),
+    )
     @patch(
         "salt.utils.thin.salt",
         type(str("salt"), (), {"__file__": "/site-packages/salt"}),
@@ -483,6 +512,7 @@ class SSHThinTestCase(TestCase):
         :return:
         """
         base_tops = [
+            "/site-packages/distro",
             "/site-packages/salt",
             "/site-packages/jinja2",
             "/site-packages/yaml",
@@ -511,6 +541,10 @@ class SSHThinTestCase(TestCase):
         self.assertEqual(len(tops), len(base_tops))
         self.assertListEqual(sorted(tops), sorted(base_tops))
 
+    @patch(
+        "salt.utils.thin.distro",
+        type("distro", (), {"__file__": "/site-packages/distro"}),
+    )
     @patch(
         "salt.utils.thin.salt",
         type(str("salt"), (), {"__file__": "/site-packages/salt"}),
@@ -566,6 +600,7 @@ class SSHThinTestCase(TestCase):
         :return:
         """
         base_tops = [
+            "/site-packages/distro",
             "/site-packages/salt",
             "/site-packages/jinja2",
             "/site-packages/yaml",
@@ -1021,6 +1056,7 @@ class SSHThinTestCase(TestCase):
             self._popen(
                 None,
                 side_effect=[
+                    (bts("distro.py"), bts("")),
                     (bts("jinja2/__init__.py"), bts("")),
                     (bts("yaml/__init__.py"), bts("")),
                     (bts("tornado/__init__.py"), bts("")),
@@ -1056,6 +1092,7 @@ class SSHThinTestCase(TestCase):
             self._popen(
                 None,
                 side_effect=[
+                    (bts("distro.py"), bts("")),
                     (bts("tornado/__init__.py"), bts("")),
                     (bts("msgpack/__init__.py"), bts("")),
                     (bts("certifi/__init__.py"), bts("")),
@@ -1089,6 +1126,7 @@ class SSHThinTestCase(TestCase):
             self._popen(
                 None,
                 side_effect=[
+                    (bts(self.fake_libs["distro"]), bts("")),
                     (bts(self.fake_libs["yaml"]), bts("")),
                     (bts(self.fake_libs["tornado"]), bts("")),
                     (bts(self.fake_libs["msgpack"]), bts("")),