Merge branch 'develop' into stable

.drone.jsonnet

@@ -1,28 +1,36 @@
 local git_suites = [
-  { name: 'Py2 2017.7(Git)', slug: 'py2-git-2017-7', depends: [] },
+  { name: 'Py2 2018.3(Git)', slug: 'py2-git-2018-3', depends: [] },
-  { name: 'Py2 2018.3(Git)', slug: 'py2-git-2018-3', depends: ['Py2 2017.7(Git)'] },
   { name: 'Py2 2019.2(Git)', slug: 'py2-git-2019-2', depends: ['Py2 2018.3(Git)'] },
   // {name: 'Py2 develop(Stable)', slug: 'py2-git-develop'},  // Don't test against Salt's develop branch. Stability is not assured.
 ];
 
+local git_py3_suites = [
+  { name: 'Py3 2018.3(Git)', slug: 'py3-git-2018-3', depends: [] },
+  { name: 'Py3 2019.2(Git)', slug: 'py3-git-2019-2', depends: ['Py3 2018.3(Git)'] },
+];
+
 local stable_suites = [
-  { name: 'Py2 2017.7(Stable)', slug: 'py2-stable-2017-7', depends: ['Py2 2017.7(Git)'] },
   { name: 'Py2 2018.3(Stable)', slug: 'py2-stable-2018-3', depends: ['Py2 2018.3(Git)'] },
   { name: 'Py2 2019.2(Stable)', slug: 'py2-stable-2019-2', depends: ['Py2 2019.2(Git)'] },
 ];
 
+local stable_py3_suites = [
+  { name: 'Py3 2018.3(Stable)', slug: 'py3-stable-2018-3', depends: ['Py3 2018.3(Git)'] },
+  { name: 'Py3 2019.2(Stable)', slug: 'py3-stable-2019-2', depends: ['Py3 2019.2(Git)'] },
+];
+
 local distros = [
   { name: 'Arch', slug: 'arch', multiplier: 0, depends: [] },
   // { name: 'Amazon 1', slug: 'amazon-1', multiplier: 1, depends: [] },
   // { name: 'Amazon 2', slug: 'amazon-2', multiplier: 2, depends: [] },
   { name: 'CentOS 6', slug: 'centos-6', multiplier: 3, depends: [] },
   { name: 'CentOS 7', slug: 'centos-7', multiplier: 4, depends: [] },
+  { name: 'CentOS 8', slug: 'centos-8', multiplier: 4, depends: [] },
   { name: 'Debian 8', slug: 'debian-8', multiplier: 5, depends: [] },
   { name: 'Debian 9', slug: 'debian-9', multiplier: 6, depends: [] },
-  { name: 'Fedora 28', slug: 'fedora-28', multiplier: 6, depends: [] },
+  { name: 'Debian 10', slug: 'debian-10', multiplier: 6, depends: [] },
-  { name: 'Fedora 29', slug: 'fedora-29', multiplier: 5, depends: [] },
+  { name: 'Fedora 30', slug: 'fedora-30', multiplier: 6, depends: [] },
   { name: 'Opensuse 15.0', slug: 'opensuse-15', multiplier: 4, depends: [] },
-  { name: 'Opensuse 42.3', slug: 'opensuse-42', multiplier: 3, depends: [] },
   { name: 'Ubuntu 16.04', slug: 'ubuntu-1604', multiplier: 1, depends: [] },
   { name: 'Ubuntu 18.04', slug: 'ubuntu-1804', multiplier: 0, depends: [] },
 ];
@@ -32,12 +40,35 @@ local stable_distros = [
   'amazon-2',
   'centos-6',
   'centos-7',
+  'centos-8',
   'debian-8',
   'debian-9',
+  'debian-10',
   'ubuntu-1604',
   'ubuntu-1804',
 ];
 
+local py3_distros = [
+  'amazon-2',
+  'centos-7',
+  'centos-8',
+  'debian-9',
+  'debian-10',
+  'ubuntu-1604',
+  'ubuntu-1804',
+  'fedora-30',
+];
+
+local py2_blacklist = [
+  'centos-8',
+  'debian-10',
+];
+
+local blacklist_2018 = [
+  'centos-8',
+  'debian-10',
+];
+
 local Shellcheck() = {
   kind: 'pipeline',
   name: 'Lint',
@@ -61,7 +92,17 @@ local Build(distro) = {
     project: 'open',
   },
 
-  local suites = if std.count(stable_distros, distro.slug) > 0 then git_suites + stable_suites else git_suites,
+  local suite =
+      if std.count(py2_blacklist, distro.slug) > 0 then
+          []
+      else if std.count(stable_distros, distro.slug) > 0 then
+          git_suites + stable_suites
+      else git_suites,
+  local suites = suite + if std.count(blacklist_2018, distro.slug) > 0 then
+                             git_py3_suites + stable_py3_suites[1:]
+                         else if std.count(py3_distros, distro.slug) > 0 then
+                             git_py3_suites + stable_py3_suites
+                         else [],
 
   steps: [
     {

.gitignore

@@ -6,5 +6,6 @@
 .idea
 
 # test-kitchen
+.kitchen
 .bundle
 Gemfile.lock

.kitchen.yml

@@ -47,6 +47,9 @@ platforms:
       provision_command:
         - pacman -Syu --noconfirm --needed systemd grep awk procps which
         - systemctl enable sshd
+  - name: centos-8
+    driver_config:
+      run_command: /usr/lib/systemd/systemd
   - name: centos-7
     driver_config:
       run_command: /usr/lib/systemd/systemd
@@ -65,13 +68,16 @@ platforms:
   - name: debian-9
     driver_config:
       run_command: /lib/systemd/systemd
-  - name: fedora-28
+  - name: debian-10
     driver_config:
-      image: fedora:28
+      run_command: /lib/systemd/systemd
+  - name: fedora-30
+    driver_config:
+      image: fedora:30
       run_command: /usr/lib/systemd/systemd
-  - name: fedora-29
+  - name: fedora-31
     driver_config:
-      image: fedora:29
+      image: fedora:31
       run_command: /usr/lib/systemd/systemd
   - name: opensuse-15
     driver_config:
@@ -82,13 +88,6 @@ platforms:
         - systemctl enable sshd.service
     provisioner:
       salt_bootstrap_options: -MPfq -y -x python2 git %s
-  - name: opensuse-42
-    driver_config:
-      image: opensuse/leap:42.3
-      run_command: /usr/lib/systemd/systemd
-      provision_command:
-        - zypper --non-interactive install --auto-agree-with-licenses dbus-1
-        - systemctl enable sshd.service
   - name: ubuntu-18.04
     driver_config:
       run_command: /lib/systemd/systemd
@@ -97,45 +96,66 @@ platforms:
       run_command: /lib/systemd/systemd
 
 suites:
-  - name: py2-git-2017-7
-    provisioner:
-      salt_version: 2017.7
   - name: py2-git-2018-3
     provisioner:
       salt_version: 2018.3
   - name: py2-git-2019-2
     provisioner:
       salt_version: 2019.2
-  - name: py2-stable-2017-7
+  - name: py2-git-master
     provisioner:
-      salt_version: 2017.7
+      salt_version: master
-      salt_bootstrap_options: -MP stable %s
-    excludes:
-      - arch
-      - fedora-28
-      - fedora-29
-      - opensuse-15
-      - opensuse-42
   - name: py2-stable-2018-3
     provisioner:
       salt_version: 2018.3
       salt_bootstrap_options: -MP stable %s
     excludes:
       - arch
-      - fedora-28
-      - fedora-29
       - opensuse-15
-      - opensuse-42
   - name: py2-stable-2019-2
     provisioner:
       salt_version: 2019.2
       salt_bootstrap_options: -MP stable %s
     excludes:
       - arch
-      - fedora-28
-      - fedora-29
       - opensuse-15
-      - opensuse-42
+  - name: py3-git-2018-3
+    provisioner:
+      salt_version: 2018.3
+      salt_bootstrap_options: -x python3 -MPfq git %s
+    excludes:
+      - amazon-1
+      - centos-6
+      - debian-8
+      - opensuse-15
+  - name: py3-git-2019-2
+    provisioner:
+      salt_version: 2019.2
+      salt_bootstrap_options: -x python3 -MPfq git %s
+    excludes:
+      - amazon-1
+      - centos-6
+      - debian-8
+      - opensuse-15
+  - name: py3-stable-2018-3
+    provisioner:
+      salt_version: 2018.3
+      salt_bootstrap_options: -x python3 -MP stable %s
+    excludes:
+      - amazon-1
+      - centos-6
+      - debian-8
+      - opensuse-15
+  - name: py3-stable-2019-2
+    provisioner:
+      salt_version: 2019.2
+      salt_bootstrap_options: -x python3 -MP stable %s
+    excludes:
+      - amazon-1
+      - centos-6
+      - debian-8
+      - opensuse-15
+      - arch
 
 verifier:
   name: shell

AUTHORS.rst

@@ -89,6 +89,7 @@ Justin Findlay              jfindlay               jfindlay@gmail.com
 Karl Grzeszczak             karlgrz
 Kenneth Wilke               KennethWilke
 Kevin Quinn                 kevinquinnyo           kevin.quinn@totalserversolutions.com
+kiemlicz                    kiemlicz
 Ky-Anh Huynh                icy
 Liu Xiaohui                 oreh                   herolxh@gmail.com
 lomeroe                     lomeroe
@@ -150,6 +151,7 @@ Shane Lee                   twangboy               slee@saltstack.com
 Shawn Butts                 shawnbutts
 Skyler Berg                 skylerberg             skylertheberg@gmail.com
 Stanislav B                 stanislavb
+stanzgy                     stanzgy                stanzgy@gmail.com
 Steve Groesz                wolfpackmars2          wolfpackmars2@yahoo.com
 Sven R                      hackacad               admin@hackacad.net
 sybix                       sybix

ChangeLog

@@ -1,5 +1,11 @@
 Version TBD (In Progress on the Develop Branch):
 
+Version 2019.11.04:
+	* Fix busybox mktemp compatibility (stanzgy) #1369
+	* Install debian 10 packages on debian 10 instead of 9 (kiemlicz) #1375
+	* move centos to python36, use python specified by -x (Ch3LL,bryceml) #1380
+	* Add debian 10 git install support (Ch3LL) #1378
+
 Version 2019.10.03:
     * Fix possible typo with `gnupg-curl` vs `gnupg curl` (zahiar)
     * Install only python3 packges if requested on ubuntu (noelmcloughlin) #1356

README.rst

@@ -28,6 +28,7 @@ sum** of the downloaded ``bootstrap-salt.sh`` file.
 
 The SHA256 sum of the ``bootstrap-salt.sh`` file, per release, is:
 
+- 2019.10.03: ``34f196f06d586ce9e1b9907660ea6e67caf57abcecfea66e0343697e3fd0d17d``
 - 2019.05.20: ``46fb5e4b7815efafd69fd703f033fe86e7b584b6770f7e0b936995bcae1cedd8``
 - 2019.02.27: ``23728e4b5e54f564062070e3be53c5602b55c24c9a76671968abbf3d609258cb``
 - 2019.01.08: ``ab7f29b75711da4bb79aff98d46654f910d569ebe3e908753a3c5119017bb163``

bootstrap-salt.sh

@@ -23,7 +23,7 @@
 #======================================================================================================================
 set -o nounset                              # Treat unset variables as an error
 
-__ScriptVersion="2019.10.03"
+__ScriptVersion="2019.11.04"
 __ScriptName="bootstrap-salt.sh"
 
 __ScriptFullName="$0"
@@ -499,7 +499,7 @@ exec 2>"$LOGPIPE"
 #              14               SIGALRM
 #              15               SIGTERM
 #----------------------------------------------------------------------------------------------------------------------
-APT_ERR=$(mktemp /tmp/apt_error.XXXX)
+APT_ERR=$(mktemp /tmp/apt_error.XXXXXX)
 __exit_cleanup() {
     EXIT_CODE=$?
 
@@ -891,7 +891,6 @@ __derive_debian_numeric_version() {
         elif [ "$INPUT_VERSION" = "stretch/sid" ]; then
             NUMERIC_VERSION=$(__parse_version_string "9.0")
         elif [ "$INPUT_VERSION" = "buster/sid" ]; then
-            # Let's start detecting the upcoming Debian 10 (Buster) release
             NUMERIC_VERSION=$(__parse_version_string "10.0")
         else
             echowarn "Unable to parse the Debian Version (codename: '$INPUT_VERSION')"
@@ -1791,7 +1790,7 @@ elif [ "${DISTRO_NAME_L}" = "debian" ]; then
   __debian_codename_translation
 fi
 
-if [ "$(echo "${DISTRO_NAME_L}" | grep -E '(debian|ubuntu|centos|red_hat|oracle|scientific|amazon)')" = "" ] && [ "$ITYPE" = "stable" ] && [ "$STABLE_REV" != "latest" ]; then
+if [ "$(echo "${DISTRO_NAME_L}" | grep -E '(debian|ubuntu|centos|red_hat|oracle|scientific|amazon|fedora)')" = "" ] && [ "$ITYPE" = "stable" ] && [ "$STABLE_REV" != "latest" ]; then
     echoerror "${DISTRO_NAME} does not have major version pegged packages support"
     exit 1
 fi
@@ -2276,8 +2275,10 @@ __overwriteconfig() {
         tempfile="/tmp/salt-config-$$"
     fi
 
+    if [ -n "$_PY_EXE" ]; then
+        good_python="$_PY_EXE"
     # If python does not have yaml installed we're on Arch and should use python2
-    if python -c "import yaml" 2> /dev/null; then
+    elif python -c "import yaml" 2> /dev/null; then
         good_python=python
     else
         good_python=python2
@@ -2517,9 +2518,16 @@ __install_pip_pkgs() {
 
     # Install pip and pip dependencies
     if ! __check_command_exists "${_pip_cmd} --version"; then
-        __PACKAGES="${_py_pkg}-setuptools ${_py_pkg}-pip gcc ${_py_pkg}-devel"
+        __PACKAGES="${_py_pkg}-setuptools ${_py_pkg}-pip gcc"
         # shellcheck disable=SC2086
-        __yum_install_noinput ${__PACKAGES} || return 1
+        if [ "$DISTRO_NAME_L" = "debian" ];then
+            __PACKAGES="${__PACKAGES} ${_py_pkg}-dev"
+            __apt_get_install_noinput ${__PACKAGES} || return 1
+        else
+            __PACKAGES="${__PACKAGES} ${_py_pkg}-devel"
+            __yum_install_noinput ${__PACKAGES} || return 1
+        fi
+
     fi
 
     echoinfo "Installing pip packages: ${_pip_pkgs} using ${_py_exe}"
@@ -2527,6 +2535,18 @@ __install_pip_pkgs() {
     ${_pip_cmd} install ${_pip_pkgs} || return 1
 }
 
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  __install_tornado_pip
+#    PARAMETERS:  python executable
+#   DESCRIPTION:  Return 0 or 1 if successfully able to install tornado<5.0
+#----------------------------------------------------------------------------------------------------------------------
+__install_tornado_pip() {
+    # OS needs tornado <5.0 from pip
+    __check_pip_allowed "You need to allow pip based installations (-P) for Tornado <5.0 in order to install Salt on Python 3"
+    ## install pip if its not installed and install tornado
+    __install_pip_pkgs "tornado<5.0" "${1}" || return 1
+}
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  __install_pip_deps
 #   DESCRIPTION:  Return 0 or 1 if successfully able to install pip packages via requirements file
@@ -3059,17 +3079,8 @@ install_ubuntu_check_services() {
 #   Debian Install Functions
 #
 __install_saltstack_debian_repository() {
-    if [ "$DISTRO_MAJOR_VERSION" -eq 10 ]; then
+    DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION"
-        # Packages for Debian 10 at repo.saltstack.com are not yet available
+    DEBIAN_CODENAME="$DISTRO_CODENAME"
-        # Set up repository for Debian 9 for Debian 10 for now until support
-        # is available at repo.saltstack.com for Debian 10.
-        echowarn "Debian 10 distribution detected, but stable packages requested. Trying packages from Debian 9. You may experience problems."
-        DEBIAN_RELEASE="9"
-        DEBIAN_CODENAME="stretch"
-    else
-        DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION"
-        DEBIAN_CODENAME="$DISTRO_CODENAME"
-    fi
 
     __PY_VERSION_REPO="apt"
     if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
@@ -3150,6 +3161,24 @@ install_debian_deps() {
     return 0
 }
 
+install_debian_git_pre() {
+    if ! __check_command_exists git; then
+        __apt_get_install_noinput git || return 1
+    fi
+
+    if [ "$_INSECURE_DL" -eq $BS_FALSE ] && [ "${_SALT_REPO_URL%%://*}" = "https" ]; then
+        __apt_get_install_noinput ca-certificates
+    fi
+
+    __git_clone_and_checkout || return 1
+
+    # Let's trigger config_salt()
+    if [ "$_TEMP_CONFIG_DIR" = "null" ]; then
+        _TEMP_CONFIG_DIR="${_SALT_GIT_CHECKOUT_DIR}/conf/"
+        CONFIG_SALT_FUNC="config_salt"
+    fi
+}
+
 install_debian_git_deps() {
     if ! __check_command_exists git; then
         __apt_get_install_noinput git || return 1
@@ -3299,7 +3328,25 @@ install_debian_9_git_deps() {
 }
 
 install_debian_10_git_deps() {
-    install_debian_9_git_deps || return 1
+    install_debian_git_pre || return 1
+
+    if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
+        _py=${_PY_EXE}
+        PY_PKG_VER=3
+        __PACKAGES="python${PY_PKG_VER}-distutils"
+    else
+        _py="python"
+        PY_PKG_VER=""
+        __PACKAGES=""
+    fi
+
+    __install_tornado_pip ${_py}|| return 1
+    __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-jinja2"
+    __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-tornado python${PY_PKG_VER}-yaml python${PY_PKG_VER}-zmq"
+
+    # shellcheck disable=SC2086
+    __apt_get_install_noinput ${__PACKAGES} || return 1
+
     return 0
 }
 
@@ -3751,13 +3798,33 @@ install_centos_stable_deps() {
         __install_saltstack_rhel_repository || return 1
     fi
 
-    __PACKAGES="yum-utils chkconfig"
+    if [ "$DISTRO_MAJOR_VERSION" -ge 8 ]; then
-
+        __PACKAGES="dnf-utils chkconfig"
-    # YAML module is used for generating custom master/minion configs
-    if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
-        __PACKAGES="${__PACKAGES} python34-PyYAML"
     else
-        __PACKAGES="${__PACKAGES} PyYAML"
+        __PACKAGES="yum-utils chkconfig"
+    fi
+
+    if [ "$DISTRO_MAJOR_VERSION" -ge 8 ]; then
+        # YAML module is used for generating custom master/minion configs
+        if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
+            __PACKAGES="${__PACKAGES} python3-pyyaml"
+        else
+            __PACKAGES="${__PACKAGES} python2-pyyaml"
+        fi
+    elif [ "$DISTRO_MAJOR_VERSION" -eq 7 ]; then
+        # YAML module is used for generating custom master/minion configs
+        if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
+            __PACKAGES="${__PACKAGES} python36-PyYAML"
+        else
+            __PACKAGES="${__PACKAGES} PyYAML"
+        fi
+    else
+        # YAML module is used for generating custom master/minion configs
+        if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
+            __PACKAGES="${__PACKAGES} python34-PyYAML"
+        else
+            __PACKAGES="${__PACKAGES} PyYAML"
+        fi
     fi
 
     # shellcheck disable=SC2086
@@ -3839,12 +3906,26 @@ install_centos_git_deps() {
 
     __git_clone_and_checkout || return 1
 
-    __PACKAGES="m2crypto"
 
+    __PACKAGES=""
+    _install_m2crypto_req=false
     if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
-        # Packages are named python34-<whatever>
+        _py=${_PY_EXE}
-        PY_PKG_VER=34
+        if [ "$DISTRO_MAJOR_VERSION" -gt 6 ]; then
+            _install_m2crypto_req=true
+        fi
+        if [ "$DISTRO_MAJOR_VERSION" -ge 8 ]; then
+            # Packages are named python3-<whatever>
+            PY_PKG_VER=3
+        else
+            # Packages are named python36-<whatever>
+            PY_PKG_VER=36
+        fi
     else
+        if [ "$DISTRO_MAJOR_VERSION" -eq 6 ]; then
+            _install_m2crypto_req=true
+        fi
+        _py="python"
         PY_PKG_VER=""
 
         # Only Py2 needs python-futures
@@ -3856,7 +3937,14 @@ install_centos_git_deps() {
         fi
     fi
 
-    __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2"
+    if [ "$DISTRO_MAJOR_VERSION" -ge 8 ]; then
+        __install_tornado_pip ${_py} || return 1
+        __PACKAGES="${__PACKAGES} python3-m2crypto"
+    else
+        __PACKAGES="${__PACKAGES} m2crypto python${PY_PKG_VER}-crypto"
+    fi
+
+    __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-jinja2"
     __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests"
     __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-tornado python${PY_PKG_VER}-zmq"
 
@@ -3874,7 +3962,7 @@ install_centos_git_deps() {
         _PIP_PACKAGES="m2crypto!=0.33.0 jinja2 msgpack-python pycrypto PyYAML tornado<5.0 zmq futures>=2.0"
 
         # install swig and openssl on cent6
-        if [ "$DISTRO_MAJOR_VERSION" -eq 6 ]; then
+        if $_install_m2crypto_req; then
             __yum_install_noinput openssl-devel swig || return 1
         fi