Merge pull request #1515 from bryceml/update_oses

update tested oses

.github/workflows/scripts/update-release-shasum.py

@@ -33,5 +33,5 @@ def main(version, sha256sum):
         README_PATH.write_text(out_contents)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main(sys.argv[1], sys.argv[2])

.github/workflows/templates/generate.py

@@ -6,190 +6,186 @@ import datetime
 os.chdir(os.path.abspath(os.path.dirname(__file__)))
 
 LINUX_DISTROS = [
-#    'amazon-1',
+    #'amazon-1',
-    'amazon-2',
+    "amazon-2",
-    'arch',
+    "arch",
-    'centos-6',
+    "centos-7",
-    'centos-7',
+    "centos-8",
-    'centos-8',
+    "debian-10",
-    'debian-10',
+    "debian-9",
-    'debian-8',
+    "fedora-32",
-    'debian-9',
+    "fedora-33",
-    'fedora-30',
+    "gentoo",
-    'fedora-31',
+    "gentoo-systemd",
-    'fedora-32',
+    "opensuse-15",
-    'gentoo',
+    "ubuntu-1604",
-    'gentoo-systemd',
+    "ubuntu-1804",
-    'opensuse-15',
+    "ubuntu-2004",
-    'ubuntu-1604',
-    'ubuntu-1804',
-    'ubuntu-2004',
 ]
 OSX = WINDOWS = []
 
 STABLE_DISTROS = [
-    'amazon-1',
+    "amazon-1",
-    'amazon-2',
+    "amazon-2",
-    'centos-6',
+    "centos-7",
-    'centos-7',
+    "centos-8",
-    'centos-8',
+    "debian-10",
-    'debian-10',
+    "debian-9",
-    'debian-8',
+    "fedora-32",
-    'debian-9',
+    "fedora-33",
-    'fedora-30',
+    "gentoo",
-    'fedora-31',
+    "gentoo-systemd",
-    'fedora-32',
+    "ubuntu-1604",
-    'gentoo',
+    "ubuntu-1804",
-    'gentoo-systemd',
+    "ubuntu-2004",
-    'ubuntu-1604',
-    'ubuntu-1804',
-    'ubuntu-2004',
 ]
 
 PY2_BLACKLIST = [
-    'centos-8',
+    "centos-8",
-    'debian-10',
+    "debian-10",
-    'fedora-30',
+    "fedora-32",
-    'fedora-31',
+    "fedora-33",
-    'fedora-32',
+    "gentoo",
-    'gentoo',
+    "gentoo-systemd",
-    'gentoo-systemd',
+    "opensuse-15",
-    'opensuse-15',
+    "ubuntu-2004",
-    'ubuntu-2004',
 ]
 
 PY3_BLACKLIST = [
-    'amazon-1',
+    "amazon-1",
-    'centos-6',
-    'debian-8',
 ]
 
 BLACKLIST_2019 = [
-    'ubuntu-2004',
+    "fedora-33",
+    "ubuntu-2004",
 ]
 
 BLACKLIST_3000 = [
-    'ubuntu-2004',
+    "fedora-33",
+    "ubuntu-2004",
 ]
 
 SALT_BRANCHES = [
-    '2019-2',
+    "2019-2",
-    '3000',
+    "3000",
-    '3001',
+    "3001",
-    '3001-0',
+    "3001-0",
-    'master',
+    "3002",
-    'latest'
+    "3002-0",
-]
+    "master",
-
+    "latest",
-SALT_POST_3000_BLACKLIST = [
-    'centos-6',
-    'debian-8',
-    'fedora-30',
 ]
 
 BRANCH_DISPLAY_NAMES = {
-    '2019-2': 'v2019.2',
+    "2019-2": "v2019.2",
-    '3000': 'v3000',
+    "3000": "v3000",
-    '3001': 'v3001',
+    "3001": "v3001",
-    '3001-0': 'v3001.0',
+    "3001-0": "v3001.0",
-    'master': 'Master',
+    "3002": "v3002",
-    'latest': 'Latest'
+    "3002-0": "v3002.0",
+    "master": "Master",
+    "latest": "Latest",
 }
 
-STABLE_BRANCH_BLACKLIST = [
+STABLE_BRANCH_BLACKLIST = []
-]
 
 LATEST_PKG_BLACKLIST = [
-    'arch',         # No packages are built
+    "arch",  # No packages are built
 ]
 
 DISTRO_DISPLAY_NAMES = {
-    'amazon-1': 'Amazon 1',
+    "amazon-1": "Amazon 1",
-    'amazon-2': 'Amazon 2',
+    "amazon-2": "Amazon 2",
-    'arch': 'Arch',
+    "arch": "Arch",
-    'centos-6': 'CentOS 6',
+    "centos-7": "CentOS 7",
-    'centos-7': 'CentOS 7',
+    "centos-8": "CentOS 8",
-    'centos-8': 'CentOS 8',
+    "debian-10": "Debian 10",
-    'debian-10': 'Debian 10',
+    "debian-9": "Debian 9",
-    'debian-8': 'Debian 8',
+    "fedora-32": "Fedora 32",
-    'debian-9': 'Debian 9',
+    "fedora-33": "Fedora 33",
-    'fedora-30': 'Fedora 30',
+    "gentoo": "Gentoo",
-    'fedora-31': 'Fedora 31',
+    "gentoo-systemd": "Gentoo (systemd)",
-    'fedora-32': 'Fedora 32',
+    "opensuse-15": "Opensuse 15",
-    'gentoo': 'Gentoo',
+    "ubuntu-1604": "Ubuntu 16.04",
-    'gentoo-systemd': 'Gentoo (systemd)',
+    "ubuntu-1804": "Ubuntu 18.04",
-    'opensuse-15': 'Opensuse 15',
+    "ubuntu-2004": "Ubuntu 20.04",
-    'ubuntu-1604': 'Ubuntu 16.04',
-    'ubuntu-1804': 'Ubuntu 18.04',
-    'ubuntu-2004': 'Ubuntu 20.04',
 }
 
 TIMEOUT_DEFAULT = 20
 TIMEOUT_OVERRIDES = {
-    'gentoo': 50,
+    "gentoo": 50,
-    'gentoo-systemd': 50,
+    "gentoo-systemd": 50,
 }
+BRANCH_ONLY_OVERRIDES = [
+    "gentoo",
+    "gentoo-systemd",
+]
+
 
 def generate_test_jobs():
-    test_jobs = ''
+    test_jobs = ""
+    branch_only_test_jobs = ""
 
     for distro in LINUX_DISTROS + OSX + WINDOWS:
-        timeout_minutes = TIMEOUT_OVERRIDES[distro] if distro in TIMEOUT_OVERRIDES else TIMEOUT_DEFAULT
+        timeout_minutes = (
+            TIMEOUT_OVERRIDES[distro]
+            if distro in TIMEOUT_OVERRIDES
+            else TIMEOUT_DEFAULT
+        )
+        needs = "    needs: lint"
+        if distro in BRANCH_ONLY_OVERRIDES:
+            needs = ""
+        current_test_jobs = ""
 
         for branch in SALT_BRANCHES:
 
-            if branch == 'master' and distro in SALT_POST_3000_BLACKLIST:
+            if branch == "latest":
-                continue
-            try:
-                if int(branch) >= 3000 and distro in SALT_POST_3000_BLACKLIST:
-                    continue
-            except ValueError:
-                pass
-
-            if branch == 'latest':
                 if distro in LATEST_PKG_BLACKLIST:
                     continue
                 if distro in LINUX_DISTROS:
-                    template = 'linux.yml'
+                    template = "linux.yml"
                 elif distro in OSX:
-                    template = 'osx.yml'
+                    template = "osx.yml"
                 elif distro in WINDOWS:
-                    template = 'windows.yml'
+                    template = "windows.yml"
                 else:
                     print("Don't know how to handle {}".format(distro))
 
                 with open(template) as rfh:
-                    test_jobs += '\n{}\n'.format(
+                    current_test_jobs += "\n{}\n".format(
-                        rfh.read().replace(
+                        rfh.read()
-                            '{python_version}-{bootstrap_type}-{branch}-{distro}',
+                        .replace(
-                            '{branch}-{distro}'
+                            "{python_version}-{bootstrap_type}-{branch}-{distro}",
-                        ).format(
+                            "{branch}-{distro}",
+                        )
+                        .format(
                             distro=distro,
                             branch=branch,
-                            display_name='{} Latest packaged release'.format(
+                            display_name="{} Latest packaged release".format(
                                 DISTRO_DISPLAY_NAMES[distro],
                             ),
-                            timeout_minutes=timeout_minutes
+                            timeout_minutes=timeout_minutes,
+                            needs=needs,
                         )
                     )
                 continue
 
-            for python_version in ('py2', 'py3'):
+            for python_version in ("py2", "py3"):
 
-                if branch == 'master' and python_version == 'py2':
+                if branch == "master" and python_version == "py2":
                     # Salt's master branch no longer supports Python 2
                     continue
 
                 try:
-                    if int(branch.split('-')[0]) >= 3000 and python_version == 'py2':
+                    if int(branch.split("-")[0]) >= 3000 and python_version == "py2":
                         # Salt's 300X versions no longer supports Python 2
                         continue
                 except ValueError:
                     pass
 
-                for bootstrap_type in ('stable', 'git'):
+                for bootstrap_type in ("stable", "git"):
-                    if bootstrap_type == 'stable':
+                    if bootstrap_type == "stable":
-                        if branch == 'master':
+                        if branch == "master":
                             # For the master branch there's no stable build
                             continue
                         if distro not in STABLE_DISTROS:
@@ -204,7 +200,7 @@ def generate_test_jobs():
 
                     if bootstrap_type == "git":
                         # .0 versions are a virtual version for pinning to the first point release of a major release, such as 3001, there is no git version.
-                        if branch.endswith('-0'):
+                        if branch.endswith("-0"):
                             continue
 
                         if python_version == "py3":
@@ -219,64 +215,85 @@ def generate_test_jobs():
                                 if branch not in allowed_branches:
                                     # Arch and Fedora default to py3.8
                                     continue
-                    if branch == '2019-2' and distro in BLACKLIST_2019:
+                    if branch == "2019-2" and distro in BLACKLIST_2019:
                         continue
 
-                    if branch == '3000' and distro in BLACKLIST_3000:
+                    if branch == "3000" and distro in BLACKLIST_3000:
                         continue
 
-                    if python_version == 'py2' and distro in PY2_BLACKLIST:
+                    if python_version == "py2" and distro in PY2_BLACKLIST:
                         continue
 
-                    if python_version == 'py3' and distro in PY3_BLACKLIST:
+                    if python_version == "py3" and distro in PY3_BLACKLIST:
                         continue
 
                     if distro in LINUX_DISTROS:
-                        template = 'linux.yml'
+                        template = "linux.yml"
                     elif distro in OSX:
-                        template = 'osx.yml'
+                        template = "osx.yml"
                     elif distro in WINDOWS:
-                        template = 'windows.yml'
+                        template = "windows.yml"
                     else:
                         print("Don't know how to handle {}".format(distro))
 
                     with open(template) as rfh:
-                        test_jobs += '\n{}\n'.format(
+                        current_test_jobs += "\n{}\n".format(
                             rfh.read().format(
                                 distro=distro,
                                 branch=branch,
                                 python_version=python_version,
                                 bootstrap_type=bootstrap_type,
-                                display_name='{} {} {} {}'.format(
+                                display_name="{} {} {} {}".format(
                                     DISTRO_DISPLAY_NAMES[distro],
                                     BRANCH_DISPLAY_NAMES[branch],
                                     python_version.capitalize(),
-                                    bootstrap_type.capitalize()
+                                    bootstrap_type.capitalize(),
                                 ),
-                                timeout_minutes=timeout_minutes
+                                timeout_minutes=timeout_minutes,
+                                needs=needs,
                             )
                         )
+        if distro in BRANCH_ONLY_OVERRIDES:
+            branch_only_test_jobs += current_test_jobs
+        else:
+            test_jobs += current_test_jobs
 
-    with open('lint.yml') as rfh:
+    with open("lint.yml") as rfh:
-        lint_job = '\n{}\n'.format(rfh.read())
+        lint_job = "\n{}\n".format(rfh.read())
 
-    with open('pre-commit.yml') as rfh:
+    with open("pre-commit.yml") as rfh:
-        pre_commit_job = '\n{}\n'.format(rfh.read())
+        pre_commit_job = "\n{}\n".format(rfh.read())
 
-    with open('../main.yml', 'w') as wfh:
+    with open("../main.yml", "w") as wfh:
-        with open('main.yml') as rfh:
+        with open("main.yml") as rfh:
             wfh.write(
-                '{}\n'.format(
+                "{}\n".format(
-                    rfh.read().format(
+                    rfh.read()
-                        jobs='{pre_commit}{lint}{test}'.format(
+                    .format(
-                            lint=lint_job,
+                        jobs="{pre_commit}{lint}{test}".format(
-                            test=test_jobs,
+                            lint=lint_job, test=test_jobs, pre_commit=pre_commit_job,
-                            pre_commit=pre_commit_job,
+                        ),
-                        )
+                        on="push, pull_request",
-                    ).strip()
+                        name="Testing",
+                    )
+                    .strip()
+                )
+            )
+
+    with open("../main-branch-only.yml", "w") as wfh:
+        with open("main.yml") as rfh:
+            wfh.write(
+                "{}\n".format(
+                    rfh.read()
+                    .format(
+                        jobs="{test}".format(test=branch_only_test_jobs,),
+                        on="push",
+                        name="Branch Testing",
+                    )
+                    .strip()
                 )
             )
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     generate_test_jobs()

.github/workflows/templates/linux.yml

@@ -3,7 +3,7 @@
     runs-on: ubuntu-latest
     timeout-minutes: {timeout_minutes}
 
-    needs: lint
+{needs}
 
     steps:
       - uses: actions/checkout@v1

.github/workflows/templates/main.yml

@@ -2,9 +2,9 @@
 #
 # This file was generated by .github/workflows/templates/generate.py
 
-name: Testing
+name: {name}
 
-on: [push, pull_request]
+on: [{on}]
 
 jobs:
 {jobs}

.kitchen.yml

@@ -57,54 +57,39 @@ platforms:
   - name: centos-7
     driver_config:
       run_command: /usr/lib/systemd/systemd
-  - name: centos-6
-    driver_config:
-      run_command: /sbin/init
-      provision_command:
-        - yum install -y upstart
-    provisioner:
-      salt_bootstrap_options: -MPfq -y -x python2.7 git %s
-  - name: debian-8
-    driver_config:
-      run_command: /lib/systemd/systemd
-      provision_command:
-        - apt-get install -y dbus
   - name: debian-9
     driver_config:
       run_command: /lib/systemd/systemd
   - name: debian-10
     driver_config:
       run_command: /lib/systemd/systemd
-  - name: fedora-30
-    driver_config:
-      image: fedora:30
-      run_command: /usr/lib/systemd/systemd
-      provision_command:
-        - dnf -y install procps-ng
-  - name: fedora-31
-    driver_config:
-      image: fedora:31
-      run_command: /usr/lib/systemd/systemd
-      provision_command:
-        - dnf -y install procps-ng
   - name: fedora-32
     driver_config:
       image: fedora:32
       run_command: /usr/lib/systemd/systemd
       provision_command:
         - dnf -y install procps-ng
+  - name: fedora-33
+    driver_config:
+      image: fedora:33
+      run_command: /usr/lib/systemd/systemd
+      provision_command:
+        - dnf -y install procps-ng
+        - sed -i 's/^PubkeyAcceptedKeyTypes.*$/&,ssh-rsa/' /etc/crypto-policies/back-ends/opensshserver.config
   - name: gentoo
     driver_config:
       image: gentoo/stage3:latest
       run_command: /sbin/init
       provision_command:
         - rc-update add sshd default
+        - emerge --update --deep --with-bdeps=y --newuse @world
   - name: gentoo-systemd
     driver_config:
       image: gentoo/stage3:systemd
       run_command: /lib/systemd/systemd
       provision_command:
         - systemctl enable sshd.service
+        - emerge --update --deep --with-bdeps=y --newuse @world
   - name: opensuse-15
     driver_config:
       image: opensuse/leap:15.1
@@ -151,9 +136,8 @@ suites:
     excludes:
       - arch
       - opensuse-15
-      - fedora-30
-      - fedora-31
       - fedora-32
+      - fedora-33
       - gentoo
       - gentoo-systemd
       - ubuntu-2004
@@ -164,9 +148,8 @@ suites:
     excludes:
       - arch
       - opensuse-15
-      - fedora-30
-      - fedora-31
       - fedora-32
+      - fedora-33
       - gentoo
       - gentoo-systemd
       - ubuntu-2004
@@ -177,8 +160,6 @@ suites:
       salt_bootstrap_options: -x python3 -MPfq git %s
     excludes:
       - amazon-1
-      - centos-6
-      - debian-8
       - ubuntu-2004
   - name: py3-git-2019-2
     provisioner:
@@ -186,8 +167,6 @@ suites:
       salt_bootstrap_options: -x python3 -MPfq git %s
     excludes:
       - amazon-1
-      - centos-6
-      - debian-8
       - ubuntu-2004
   - name: py3-git-3001
     provisioner:
@@ -195,16 +174,18 @@ suites:
       salt_bootstrap_options: -x python3 -MPfq git %s
     excludes:
       - amazon-1
-      - centos-6
+  - name: py3-git-3002
-      - debian-8
+    provisioner:
+      salt_version: 3002
+      salt_bootstrap_options: -x python3 -MPfq git %s
+    excludes:
+      - amazon-1
   - name: py3-stable-2019-2
     provisioner:
       salt_version: 2019.2
       salt_bootstrap_options: -x python3 -MP stable %s
     excludes:
       - amazon-1
-      - centos-6
-      - debian-8
       - opensuse-15
       - arch
       - ubuntu-2004
@@ -214,8 +195,6 @@ suites:
       salt_bootstrap_options: -x python3 -MP stable %s
     excludes:
       - amazon-1
-      - centos-6
-      - debian-8
       - opensuse-15
       - arch
       - ubuntu-2004
@@ -225,12 +204,9 @@ suites:
       salt_bootstrap_options: -x python3 -MP stable 3001.0
     excludes:
       - amazon-1
-      - centos-6
-      - debian-8
       - opensuse-15
-      - fedora-30
-      - fedora-31
       - fedora-32
+      - fedora-33
       - arch
   - name: py3-stable-3001
     provisioner:
@@ -238,8 +214,24 @@ suites:
       salt_bootstrap_options: -x python3 -MP stable %s
     excludes:
       - amazon-1
-      - centos-6
+      - opensuse-15
-      - debian-8
+      - arch
+  - name: py3-stable-3002-0
+    provisioner:
+      salt_version: 3002
+      salt_bootstrap_options: -x python3 -MP stable 3002.0
+    excludes:
+      - amazon-1
+      - opensuse-15
+      - fedora-32
+      - fedora-33
+      - arch
+  - name: py3-stable-3002
+    provisioner:
+      salt_version: 3002
+      salt_bootstrap_options: -x python3 -MP stable %s
+    excludes:
+      - amazon-1
       - opensuse-15
       - arch
   - name: py3-git-master
@@ -248,8 +240,6 @@ suites:
       salt_bootstrap_options: -x python3 -MPfq -D git %s
     excludes:
       - amazon-1
-      - centos-6
-      - debian-8
 
   - name: latest
     provisioner:

.pre-commit-config.yaml

@@ -11,6 +11,11 @@ repos:
         args: [--fix=lf]
       - id: end-of-file-fixer     # Makes sure files end in a newline and only a newline.
 
+  - repo: https://github.com/psf/black
+    rev: 19.10b0
+    hooks:
+      - id: black
+
   - repo: local
     hooks:
       - id: generate-actions-workflow

tests/conftest.py

@@ -7,32 +7,40 @@ import logging
 log = logging.getLogger(__name__)
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture(scope="session")
 def host():
-    if os.environ.get('KITCHEN_USERNAME') == 'vagrant':
+    if os.environ.get("KITCHEN_USERNAME") == "vagrant":
-        if 'windows' in os.environ.get('KITCHEN_INSTANCE'):
+        if "windows" in os.environ.get("KITCHEN_INSTANCE"):
             return testinfra.get_host(
-                'winrm://{KITCHEN_USERNAME}:{KITCHEN_PASSWORD}@{KITCHEN_HOSTNAME}:{KITCHEN_PORT}'.format(**os.environ),
+                "winrm://{KITCHEN_USERNAME}:{KITCHEN_PASSWORD}@{KITCHEN_HOSTNAME}:{KITCHEN_PORT}".format(
-                no_ssl=True)
+                    **os.environ
+                ),
+                no_ssl=True,
+            )
         return testinfra.get_host(
-            'paramiko://{KITCHEN_USERNAME}@{KITCHEN_HOSTNAME}:{KITCHEN_PORT}'.format(**os.environ),
+            "paramiko://{KITCHEN_USERNAME}@{KITCHEN_HOSTNAME}:{KITCHEN_PORT}".format(
-            ssh_identity_file=os.environ.get('KITCHEN_SSH_KEY'))
+                **os.environ
-    return testinfra.get_host('docker://{KITCHEN_USERNAME}@{KITCHEN_CONTAINER_ID}'.format(**os.environ))
+            ),
+            ssh_identity_file=os.environ.get("KITCHEN_SSH_KEY"),
+        )
+    return testinfra.get_host(
+        "docker://{KITCHEN_USERNAME}@{KITCHEN_CONTAINER_ID}".format(**os.environ)
+    )
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture(scope="session")
 def target_python_version():
     target_python = os.environ["KITCHEN_SUITE"].split("-", 1)[0]
     if target_python == "latest":
-        pytest.skip("Unable to get target python from {}".format(os.environ["KITCHEN_SUITE"]))
+        pytest.skip(
+            "Unable to get target python from {}".format(os.environ["KITCHEN_SUITE"])
+        )
     return int(target_python.replace("py", ""))
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture(scope="session")
 def target_salt_version():
     target_salt = os.environ["KITCHEN_SUITE"].split("-", 2)[-1].replace("-", ".")
-    if target_salt.endswith(".0") and float(target_salt) >= 3000:
-        target_salt = ".".join(target_salt.split(".")[:-1])
     if target_salt in ("latest", "master"):
         pytest.skip("Don't have a specific salt version to test against")
     return target_salt

tests/integration/test_installation.py

@@ -9,16 +9,19 @@ log = logging.getLogger(__name__)
 
 def test_ping(host):
     with host.sudo():
-        assert host.salt('test.ping', '--timeout=120')
+        assert host.salt("test.ping", "--timeout=120")
 
 
 def test_target_python_version(host, target_python_version):
     with host.sudo():
-        ret = host.salt('grains.item', 'pythonversion', '--timeout=120')
+        ret = host.salt("grains.item", "pythonversion", "--timeout=120")
         assert ret["pythonversion"][0] == target_python_version
 
 
 def test_target_salt_version(host, target_salt_version):
     with host.sudo():
-        ret = host.salt('grains.item', 'saltversion', '--timeout=120')
+        ret = host.salt("grains.item", "saltversion", "--timeout=120")
-        assert ret["saltversion"].startswith(target_salt_version)
+        if target_salt_version.endswith(".0"):
+            assert ret["saltversion"] == ".".join(target_salt_version.split(".")[:-1])
+        else:
+            assert ret["saltversion"].startswith(target_salt_version)