Merge pull request #1380 from Ch3LL/cent8_git

move centos to python36, use python specified by -x to render passed config

.drone.jsonnet

@@ -1,28 +1,36 @@
 local git_suites = [
-  { name: 'Py2 2017.7(Git)', slug: 'py2-git-2017-7', depends: [] },
-  { name: 'Py2 2018.3(Git)', slug: 'py2-git-2018-3', depends: ['Py2 2017.7(Git)'] },
+  { name: 'Py2 2018.3(Git)', slug: 'py2-git-2018-3', depends: [] },
   { name: 'Py2 2019.2(Git)', slug: 'py2-git-2019-2', depends: ['Py2 2018.3(Git)'] },
   // {name: 'Py2 develop(Stable)', slug: 'py2-git-develop'},  // Don't test against Salt's develop branch. Stability is not assured.
 ];
 
+local git_py3_suites = [
+  { name: 'Py3 2018.3(Git)', slug: 'py3-git-2018-3', depends: [] },
+  { name: 'Py3 2019.2(Git)', slug: 'py3-git-2019-2', depends: ['Py3 2018.3(Git)'] },
+];
+
 local stable_suites = [
-  { name: 'Py2 2017.7(Stable)', slug: 'py2-stable-2017-7', depends: ['Py2 2017.7(Git)'] },
   { name: 'Py2 2018.3(Stable)', slug: 'py2-stable-2018-3', depends: ['Py2 2018.3(Git)'] },
   { name: 'Py2 2019.2(Stable)', slug: 'py2-stable-2019-2', depends: ['Py2 2019.2(Git)'] },
 ];
 
+local stable_py3_suites = [
+  { name: 'Py3 2018.3(Stable)', slug: 'py3-stable-2018-3', depends: ['Py3 2018.3(Git)'] },
+  { name: 'Py3 2019.2(Stable)', slug: 'py3-stable-2019-2', depends: ['Py3 2019.2(Git)'] },
+];
+
 local distros = [
   { name: 'Arch', slug: 'arch', multiplier: 0, depends: [] },
   // { name: 'Amazon 1', slug: 'amazon-1', multiplier: 1, depends: [] },
   // { name: 'Amazon 2', slug: 'amazon-2', multiplier: 2, depends: [] },
   { name: 'CentOS 6', slug: 'centos-6', multiplier: 3, depends: [] },
   { name: 'CentOS 7', slug: 'centos-7', multiplier: 4, depends: [] },
+  { name: 'CentOS 8', slug: 'centos-8', multiplier: 4, depends: [] },
   { name: 'Debian 8', slug: 'debian-8', multiplier: 5, depends: [] },
   { name: 'Debian 9', slug: 'debian-9', multiplier: 6, depends: [] },
-  { name: 'Fedora 28', slug: 'fedora-28', multiplier: 6, depends: [] },
-  { name: 'Fedora 29', slug: 'fedora-29', multiplier: 5, depends: [] },
+  { name: 'Debian 10', slug: 'debian-10', multiplier: 6, depends: [] },
+  { name: 'Fedora 30', slug: 'fedora-30', multiplier: 6, depends: [] },
   { name: 'Opensuse 15.0', slug: 'opensuse-15', multiplier: 4, depends: [] },
-  { name: 'Opensuse 42.3', slug: 'opensuse-42', multiplier: 3, depends: [] },
   { name: 'Ubuntu 16.04', slug: 'ubuntu-1604', multiplier: 1, depends: [] },
   { name: 'Ubuntu 18.04', slug: 'ubuntu-1804', multiplier: 0, depends: [] },
 ];
@@ -32,12 +40,35 @@ local stable_distros = [
   'amazon-2',
   'centos-6',
   'centos-7',
+  'centos-8',
   'debian-8',
   'debian-9',
+  'debian-10',
   'ubuntu-1604',
   'ubuntu-1804',
 ];
 
+local py3_distros = [
+  'amazon-2',
+  'centos-7',
+  'centos-8',
+  'debian-9',
+  'debian-10',
+  'ubuntu-1604',
+  'ubuntu-1804',
+  'fedora-30',
+];
+
+local py2_blacklist = [
+  'centos-8',
+  'debian-10',
+];
+
+local blacklist_2018 = [
+  'centos-8',
+  'debian-10',
+];
+
 local Shellcheck() = {
   kind: 'pipeline',
   name: 'Lint',
@@ -61,7 +92,17 @@ local Build(distro) = {
     project: 'open',
   },
 
-  local suites = if std.count(stable_distros, distro.slug) > 0 then git_suites + stable_suites else git_suites,
+  local suite =
+      if std.count(py2_blacklist, distro.slug) > 0 then
+          []
+      else if std.count(stable_distros, distro.slug) > 0 then
+          git_suites + stable_suites
+      else git_suites,
+  local suites = suite + if std.count(blacklist_2018, distro.slug) > 0 then
+                             git_py3_suites + stable_py3_suites[1:]
+                         else if std.count(py3_distros, distro.slug) > 0 then
+                             git_py3_suites + stable_py3_suites
+                         else [],
 
   steps: [
     {

.gitignore

@@ -6,5 +6,6 @@
 .idea
 
 # test-kitchen
+.kitchen
 .bundle
 Gemfile.lock

.kitchen.yml

@@ -47,6 +47,9 @@ platforms:
       provision_command:
         - pacman -Syu --noconfirm --needed systemd grep awk procps which
         - systemctl enable sshd
+  - name: centos-8
+    driver_config:
+      run_command: /usr/lib/systemd/systemd
   - name: centos-7
     driver_config:
       run_command: /usr/lib/systemd/systemd
@@ -65,13 +68,16 @@ platforms:
   - name: debian-9
     driver_config:
       run_command: /lib/systemd/systemd
-  - name: fedora-28
+  - name: debian-10
     driver_config:
-      image: fedora:28
+      run_command: /lib/systemd/systemd
+  - name: fedora-30
+    driver_config:
+      image: fedora:30
       run_command: /usr/lib/systemd/systemd
-  - name: fedora-29
+  - name: fedora-31
     driver_config:
-      image: fedora:29
+      image: fedora:31
       run_command: /usr/lib/systemd/systemd
   - name: opensuse-15
     driver_config:
@@ -82,13 +88,6 @@ platforms:
         - systemctl enable sshd.service
     provisioner:
       salt_bootstrap_options: -MPfq -y -x python2 git %s
-  - name: opensuse-42
-    driver_config:
-      image: opensuse/leap:42.3
-      run_command: /usr/lib/systemd/systemd
-      provision_command:
-        - zypper --non-interactive install --auto-agree-with-licenses dbus-1
-        - systemctl enable sshd.service
   - name: ubuntu-18.04
     driver_config:
       run_command: /lib/systemd/systemd
@@ -97,45 +96,66 @@ platforms:
       run_command: /lib/systemd/systemd
 
 suites:
-  - name: py2-git-2017-7
-    provisioner:
-      salt_version: 2017.7
   - name: py2-git-2018-3
     provisioner:
       salt_version: 2018.3
   - name: py2-git-2019-2
     provisioner:
       salt_version: 2019.2
-  - name: py2-stable-2017-7
+  - name: py2-git-master
     provisioner:
-      salt_version: 2017.7
-      salt_bootstrap_options: -MP stable %s
-    excludes:
-      - arch
-      - fedora-28
-      - fedora-29
-      - opensuse-15
-      - opensuse-42
+      salt_version: master
   - name: py2-stable-2018-3
     provisioner:
       salt_version: 2018.3
       salt_bootstrap_options: -MP stable %s
     excludes:
       - arch
-      - fedora-28
-      - fedora-29
       - opensuse-15
-      - opensuse-42
   - name: py2-stable-2019-2
     provisioner:
       salt_version: 2019.2
       salt_bootstrap_options: -MP stable %s
     excludes:
       - arch
-      - fedora-28
-      - fedora-29
       - opensuse-15
-      - opensuse-42
+  - name: py3-git-2018-3
+    provisioner:
+      salt_version: 2018.3
+      salt_bootstrap_options: -x python3 -MPfq git %s
+    excludes:
+      - amazon-1
+      - centos-6
+      - debian-8
+      - opensuse-15
+  - name: py3-git-2019-2
+    provisioner:
+      salt_version: 2019.2
+      salt_bootstrap_options: -x python3 -MPfq git %s
+    excludes:
+      - amazon-1
+      - centos-6
+      - debian-8
+      - opensuse-15
+  - name: py3-stable-2018-3
+    provisioner:
+      salt_version: 2018.3
+      salt_bootstrap_options: -x python3 -MP stable %s
+    excludes:
+      - amazon-1
+      - centos-6
+      - debian-8
+      - opensuse-15
+  - name: py3-stable-2019-2
+    provisioner:
+      salt_version: 2019.2
+      salt_bootstrap_options: -x python3 -MP stable %s
+    excludes:
+      - amazon-1
+      - centos-6
+      - debian-8
+      - opensuse-15
+      - arch
 
 verifier:
   name: shell

bootstrap-salt.sh

@@ -1790,7 +1790,7 @@ elif [ "${DISTRO_NAME_L}" = "debian" ]; then
   __debian_codename_translation
 fi
 
-if [ "$(echo "${DISTRO_NAME_L}" | grep -E '(debian|ubuntu|centos|red_hat|oracle|scientific|amazon)')" = "" ] && [ "$ITYPE" = "stable" ] && [ "$STABLE_REV" != "latest" ]; then
+if [ "$(echo "${DISTRO_NAME_L}" | grep -E '(debian|ubuntu|centos|red_hat|oracle|scientific|amazon|fedora)')" = "" ] && [ "$ITYPE" = "stable" ] && [ "$STABLE_REV" != "latest" ]; then
     echoerror "${DISTRO_NAME} does not have major version pegged packages support"
     exit 1
 fi
@@ -2275,8 +2275,10 @@ __overwriteconfig() {
         tempfile="/tmp/salt-config-$$"
     fi
 
+    if [ -n "$_PY_EXE" ]; then
+        good_python="$_PY_EXE"
     # If python does not have yaml installed we're on Arch and should use python2
-    if python -c "import yaml" 2> /dev/null; then
+    elif python -c "import yaml" 2> /dev/null; then
         good_python=python
     else
         good_python=python2
@@ -2526,6 +2528,18 @@ __install_pip_pkgs() {
     ${_pip_cmd} install ${_pip_pkgs} || return 1
 }
 
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  __install_tornado_pip
+#    PARAMETERS:  python executable
+#   DESCRIPTION:  Return 0 or 1 if successfully able to install tornado<5.0
+#----------------------------------------------------------------------------------------------------------------------
+__install_tornado_pip() {
+    # OS needs tornado <5.0 from pip
+    __check_pip_allowed "You need to allow pip based installations (-P) for Tornado <5.0 in order to install Salt on Python 3"
+    ## install pip if its not installed and install tornado
+    __install_pip_pkgs "tornado<5.0" "${1}" || return 1
+}
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  __install_pip_deps
 #   DESCRIPTION:  Return 0 or 1 if successfully able to install pip packages via requirements file
@@ -3741,13 +3755,33 @@ install_centos_stable_deps() {
         __install_saltstack_rhel_repository || return 1
     fi
 
-    __PACKAGES="yum-utils chkconfig"
-
-    # YAML module is used for generating custom master/minion configs
-    if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
-        __PACKAGES="${__PACKAGES} python34-PyYAML"
+    if [ "$DISTRO_MAJOR_VERSION" -ge 8 ]; then
+        __PACKAGES="dnf-utils chkconfig"
     else
-        __PACKAGES="${__PACKAGES} PyYAML"
+        __PACKAGES="yum-utils chkconfig"
+    fi
+
+    if [ "$DISTRO_MAJOR_VERSION" -ge 8 ]; then
+        # YAML module is used for generating custom master/minion configs
+        if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
+            __PACKAGES="${__PACKAGES} python3-pyyaml"
+        else
+            __PACKAGES="${__PACKAGES} python2-pyyaml"
+        fi
+    elif [ "$DISTRO_MAJOR_VERSION" -eq 7 ]; then
+        # YAML module is used for generating custom master/minion configs
+        if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
+            __PACKAGES="${__PACKAGES} python36-PyYAML"
+        else
+            __PACKAGES="${__PACKAGES} PyYAML"
+        fi
+    else
+        # YAML module is used for generating custom master/minion configs
+        if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
+            __PACKAGES="${__PACKAGES} python34-PyYAML"
+        else
+            __PACKAGES="${__PACKAGES} PyYAML"
+        fi
     fi
 
     # shellcheck disable=SC2086
@@ -3829,12 +3863,26 @@ install_centos_git_deps() {
 
     __git_clone_and_checkout || return 1
 
-    __PACKAGES="m2crypto"
 
+    __PACKAGES=""
+    _install_m2crypto_req=false
     if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
-        # Packages are named python34-<whatever>
-        PY_PKG_VER=34
+        _py=${_PY_EXE}
+        if [ "$DISTRO_MAJOR_VERSION" -gt 6 ]; then
+            _install_m2crypto_req=true
+        fi
+        if [ "$DISTRO_MAJOR_VERSION" -ge 8 ]; then
+            # Packages are named python3-<whatever>
+            PY_PKG_VER=3
+        else
+            # Packages are named python36-<whatever>
+            PY_PKG_VER=36
+        fi
     else
+        if [ "$DISTRO_MAJOR_VERSION" -eq 6 ]; then
+            _install_m2crypto_req=true
+        fi
+        _py="python"
         PY_PKG_VER=""
 
         # Only Py2 needs python-futures
@@ -3846,7 +3894,14 @@ install_centos_git_deps() {
         fi
     fi
 
-    __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2"
+    if [ "$DISTRO_MAJOR_VERSION" -ge 8 ]; then
+        __install_tornado_pip ${_py} || return 1
+        __PACKAGES="${__PACKAGES} python3-m2crypto"
+    else
+        __PACKAGES="${__PACKAGES} m2crypto python${PY_PKG_VER}-crypto"
+    fi
+
+    __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-jinja2"
     __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests"
     __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-tornado python${PY_PKG_VER}-zmq"
 
@@ -3864,7 +3919,7 @@ install_centos_git_deps() {
         _PIP_PACKAGES="m2crypto!=0.33.0 jinja2 msgpack-python pycrypto PyYAML tornado<5.0 zmq futures>=2.0"
 
         # install swig and openssl on cent6
-        if [ "$DISTRO_MAJOR_VERSION" -eq 6 ]; then
+        if $_install_m2crypto_req; then
             __yum_install_noinput openssl-devel swig || return 1
         fi