Merge pull request #1384 from Ch3LL/merge_stable

Merge develop into stable for 2019.11.04 release
2025-04-16 01:30:21 +00:00 · 2019-11-04 10:24:12 -05:00 · 2019-11-04 10:24:12 -05:00 · 818481e267
commit 818481e267
parent 33e831bef9 0964335792
8 changed files with 622 additions and 398 deletions
--- a/.drone.jsonnet
+++ b/.drone.jsonnet
@ -1,28 +1,36 @@
 local git_suites = [
-  { name: 'Py2 2017.7(Git)', slug: 'py2-git-2017-7', depends: [] },
-  { name: 'Py2 2018.3(Git)', slug: 'py2-git-2018-3', depends: ['Py2 2017.7(Git)'] },
+  { name: 'Py2 2018.3(Git)', slug: 'py2-git-2018-3', depends: [] },
  { name: 'Py2 2019.2(Git)', slug: 'py2-git-2019-2', depends: ['Py2 2018.3(Git)'] },
  // {name: 'Py2 develop(Stable)', slug: 'py2-git-develop'},  // Don't test against Salt's develop branch. Stability is not assured.
 ];

+local git_py3_suites = [
+  { name: 'Py3 2018.3(Git)', slug: 'py3-git-2018-3', depends: [] },
+  { name: 'Py3 2019.2(Git)', slug: 'py3-git-2019-2', depends: ['Py3 2018.3(Git)'] },
+];
+
 local stable_suites = [
-  { name: 'Py2 2017.7(Stable)', slug: 'py2-stable-2017-7', depends: ['Py2 2017.7(Git)'] },
  { name: 'Py2 2018.3(Stable)', slug: 'py2-stable-2018-3', depends: ['Py2 2018.3(Git)'] },
  { name: 'Py2 2019.2(Stable)', slug: 'py2-stable-2019-2', depends: ['Py2 2019.2(Git)'] },
 ];

+local stable_py3_suites = [
+  { name: 'Py3 2018.3(Stable)', slug: 'py3-stable-2018-3', depends: ['Py3 2018.3(Git)'] },
+  { name: 'Py3 2019.2(Stable)', slug: 'py3-stable-2019-2', depends: ['Py3 2019.2(Git)'] },
+];
+
 local distros = [
  { name: 'Arch', slug: 'arch', multiplier: 0, depends: [] },
  // { name: 'Amazon 1', slug: 'amazon-1', multiplier: 1, depends: [] },
  // { name: 'Amazon 2', slug: 'amazon-2', multiplier: 2, depends: [] },
  { name: 'CentOS 6', slug: 'centos-6', multiplier: 3, depends: [] },
  { name: 'CentOS 7', slug: 'centos-7', multiplier: 4, depends: [] },
+  { name: 'CentOS 8', slug: 'centos-8', multiplier: 4, depends: [] },
  { name: 'Debian 8', slug: 'debian-8', multiplier: 5, depends: [] },
  { name: 'Debian 9', slug: 'debian-9', multiplier: 6, depends: [] },
-  { name: 'Fedora 28', slug: 'fedora-28', multiplier: 6, depends: [] },
-  { name: 'Fedora 29', slug: 'fedora-29', multiplier: 5, depends: [] },
+  { name: 'Debian 10', slug: 'debian-10', multiplier: 6, depends: [] },
+  { name: 'Fedora 30', slug: 'fedora-30', multiplier: 6, depends: [] },
  { name: 'Opensuse 15.0', slug: 'opensuse-15', multiplier: 4, depends: [] },
-  { name: 'Opensuse 42.3', slug: 'opensuse-42', multiplier: 3, depends: [] },
  { name: 'Ubuntu 16.04', slug: 'ubuntu-1604', multiplier: 1, depends: [] },
  { name: 'Ubuntu 18.04', slug: 'ubuntu-1804', multiplier: 0, depends: [] },
 ];
@ -32,12 +40,35 @@ local stable_distros = [
  'amazon-2',
  'centos-6',
  'centos-7',
+  'centos-8',
  'debian-8',
  'debian-9',
+  'debian-10',
  'ubuntu-1604',
  'ubuntu-1804',
 ];

+local py3_distros = [
+  'amazon-2',
+  'centos-7',
+  'centos-8',
+  'debian-9',
+  'debian-10',
+  'ubuntu-1604',
+  'ubuntu-1804',
+  'fedora-30',
+];
+
+local py2_blacklist = [
+  'centos-8',
+  'debian-10',
+];
+
+local blacklist_2018 = [
+  'centos-8',
+  'debian-10',
+];
+
 local Shellcheck() = {
  kind: 'pipeline',
  name: 'Lint',
@ -61,7 +92,17 @@ local Build(distro) = {
    project: 'open',
  },

-  local suites = if std.count(stable_distros, distro.slug) > 0 then git_suites + stable_suites else git_suites,
+  local suite =
+      if std.count(py2_blacklist, distro.slug) > 0 then
+          []
+      else if std.count(stable_distros, distro.slug) > 0 then
+          git_suites + stable_suites
+      else git_suites,
+  local suites = suite + if std.count(blacklist_2018, distro.slug) > 0 then
+                             git_py3_suites + stable_py3_suites[1:]
+                         else if std.count(py3_distros, distro.slug) > 0 then
+                             git_py3_suites + stable_py3_suites
+                         else [],

  steps: [
    {
--- a/.drone.yml
+++ b/.drone.yml
--- a/.gitignore
+++ b/.gitignore
@ -6,5 +6,6 @@
 .idea

 # test-kitchen
+.kitchen
 .bundle
 Gemfile.lock
--- a/.kitchen.yml
+++ b/.kitchen.yml
@ -47,6 +47,9 @@ platforms:
      provision_command:
        - pacman -Syu --noconfirm --needed systemd grep awk procps which
        - systemctl enable sshd
+  - name: centos-8
+    driver_config:
+      run_command: /usr/lib/systemd/systemd
  - name: centos-7
    driver_config:
      run_command: /usr/lib/systemd/systemd
@ -65,13 +68,16 @@ platforms:
  - name: debian-9
    driver_config:
      run_command: /lib/systemd/systemd
-  - name: fedora-28
+  - name: debian-10
    driver_config:
-      image: fedora:28
+      run_command: /lib/systemd/systemd
+  - name: fedora-30
+    driver_config:
+      image: fedora:30
      run_command: /usr/lib/systemd/systemd
-  - name: fedora-29
+  - name: fedora-31
    driver_config:
-      image: fedora:29
+      image: fedora:31
      run_command: /usr/lib/systemd/systemd
  - name: opensuse-15
    driver_config:
@ -82,13 +88,6 @@ platforms:
        - systemctl enable sshd.service
    provisioner:
      salt_bootstrap_options: -MPfq -y -x python2 git %s
-  - name: opensuse-42
-    driver_config:
-      image: opensuse/leap:42.3
-      run_command: /usr/lib/systemd/systemd
-      provision_command:
-        - zypper --non-interactive install --auto-agree-with-licenses dbus-1
-        - systemctl enable sshd.service
  - name: ubuntu-18.04
    driver_config:
      run_command: /lib/systemd/systemd
@ -97,45 +96,66 @@ platforms:
      run_command: /lib/systemd/systemd

 suites:
-  - name: py2-git-2017-7
-    provisioner:
-      salt_version: 2017.7
  - name: py2-git-2018-3
    provisioner:
      salt_version: 2018.3
  - name: py2-git-2019-2
    provisioner:
      salt_version: 2019.2
-  - name: py2-stable-2017-7
+  - name: py2-git-master
    provisioner:
-      salt_version: 2017.7
-      salt_bootstrap_options: -MP stable %s
-    excludes:
-      - arch
-      - fedora-28
-      - fedora-29
-      - opensuse-15
-      - opensuse-42
+      salt_version: master
  - name: py2-stable-2018-3
    provisioner:
      salt_version: 2018.3
      salt_bootstrap_options: -MP stable %s
    excludes:
      - arch
-      - fedora-28
-      - fedora-29
      - opensuse-15
-      - opensuse-42
  - name: py2-stable-2019-2
    provisioner:
      salt_version: 2019.2
      salt_bootstrap_options: -MP stable %s
    excludes:
      - arch
-      - fedora-28
-      - fedora-29
      - opensuse-15
-      - opensuse-42
+  - name: py3-git-2018-3
+    provisioner:
+      salt_version: 2018.3
+      salt_bootstrap_options: -x python3 -MPfq git %s
+    excludes:
+      - amazon-1
+      - centos-6
+      - debian-8
+      - opensuse-15
+  - name: py3-git-2019-2
+    provisioner:
+      salt_version: 2019.2
+      salt_bootstrap_options: -x python3 -MPfq git %s
+    excludes:
+      - amazon-1
+      - centos-6
+      - debian-8
+      - opensuse-15
+  - name: py3-stable-2018-3
+    provisioner:
+      salt_version: 2018.3
+      salt_bootstrap_options: -x python3 -MP stable %s
+    excludes:
+      - amazon-1
+      - centos-6
+      - debian-8
+      - opensuse-15
+  - name: py3-stable-2019-2
+    provisioner:
+      salt_version: 2019.2
+      salt_bootstrap_options: -x python3 -MP stable %s
+    excludes:
+      - amazon-1
+      - centos-6
+      - debian-8
+      - opensuse-15
+      - arch

 verifier:
  name: shell
--- a/AUTHORS.rst
+++ b/AUTHORS.rst
@ -89,6 +89,7 @@ Justin Findlay              jfindlay               jfindlay@gmail.com
 Karl Grzeszczak             karlgrz
 Kenneth Wilke               KennethWilke
 Kevin Quinn                 kevinquinnyo           kevin.quinn@totalserversolutions.com
+kiemlicz                    kiemlicz
 Ky-Anh Huynh                icy
 Liu Xiaohui                 oreh                   herolxh@gmail.com
 lomeroe                     lomeroe
@ -150,6 +151,7 @@ Shane Lee                   twangboy               slee@saltstack.com
 Shawn Butts                 shawnbutts
 Skyler Berg                 skylerberg             skylertheberg@gmail.com
 Stanislav B                 stanislavb
+stanzgy                     stanzgy                stanzgy@gmail.com
 Steve Groesz                wolfpackmars2          wolfpackmars2@yahoo.com
 Sven R                      hackacad               admin@hackacad.net
 sybix                       sybix
--- a/6
+++ b/6
@ -1,5 +1,11 @@
 Version TBD (In Progress on the Develop Branch):

+Version 2019.11.04:
+	* Fix busybox mktemp compatibility (stanzgy) #1369
+	* Install debian 10 packages on debian 10 instead of 9 (kiemlicz) #1375
+	* move centos to python36, use python specified by -x (Ch3LL,bryceml) #1380
+	* Add debian 10 git install support (Ch3LL) #1378
+
 Version 2019.10.03:
    * Fix possible typo with `gnupg-curl` vs `gnupg curl` (zahiar)
    * Install only python3 packges if requested on ubuntu (noelmcloughlin) #1356
--- a/README.rst
+++ b/README.rst
@ -28,6 +28,7 @@ sum** of the downloaded ``bootstrap-salt.sh`` file.

 The SHA256 sum of the ``bootstrap-salt.sh`` file, per release, is:

+- 2019.10.03: ``34f196f06d586ce9e1b9907660ea6e67caf57abcecfea66e0343697e3fd0d17d``
 - 2019.05.20: ``46fb5e4b7815efafd69fd703f033fe86e7b584b6770f7e0b936995bcae1cedd8``
 - 2019.02.27: ``23728e4b5e54f564062070e3be53c5602b55c24c9a76671968abbf3d609258cb``
 - 2019.01.08: ``ab7f29b75711da4bb79aff98d46654f910d569ebe3e908753a3c5119017bb163``
--- a/bootstrap-salt.sh
+++ b/bootstrap-salt.sh
@ -23,7 +23,7 @@
 #======================================================================================================================
 set -o nounset                              # Treat unset variables as an error

-__ScriptVersion="2019.10.03"
+__ScriptVersion="2019.11.04"
 __ScriptName="bootstrap-salt.sh"

 __ScriptFullName="$0"
@ -499,7 +499,7 @@ exec 2>"$LOGPIPE"
 #              14               SIGALRM
 #              15               SIGTERM
 #----------------------------------------------------------------------------------------------------------------------
-APT_ERR=$(mktemp /tmp/apt_error.XXXX)
+APT_ERR=$(mktemp /tmp/apt_error.XXXXXX)
 __exit_cleanup() {
    EXIT_CODE=$?

@ -891,7 +891,6 @@ __derive_debian_numeric_version() {
        elif [ "$INPUT_VERSION" = "stretch/sid" ]; then
            NUMERIC_VERSION=$(__parse_version_string "9.0")
        elif [ "$INPUT_VERSION" = "buster/sid" ]; then
-            # Let's start detecting the upcoming Debian 10 (Buster) release
            NUMERIC_VERSION=$(__parse_version_string "10.0")
        else
            echowarn "Unable to parse the Debian Version (codename: '$INPUT_VERSION')"
@ -1791,7 +1790,7 @@ elif [ "${DISTRO_NAME_L}" = "debian" ]; then
  __debian_codename_translation
 fi

-if [ "$(echo "${DISTRO_NAME_L}" | grep -E '(debian|ubuntu|centos|red_hat|oracle|scientific|amazon)')" = "" ] && [ "$ITYPE" = "stable" ] && [ "$STABLE_REV" != "latest" ]; then
+if [ "$(echo "${DISTRO_NAME_L}" | grep -E '(debian|ubuntu|centos|red_hat|oracle|scientific|amazon|fedora)')" = "" ] && [ "$ITYPE" = "stable" ] && [ "$STABLE_REV" != "latest" ]; then
    echoerror "${DISTRO_NAME} does not have major version pegged packages support"
    exit 1
 fi
@ -2276,8 +2275,10 @@ __overwriteconfig() {
        tempfile="/tmp/salt-config-$$"
    fi

+    if [ -n "$_PY_EXE" ]; then
+        good_python="$_PY_EXE"
    # If python does not have yaml installed we're on Arch and should use python2
-    if python -c "import yaml" 2> /dev/null; then
+    elif python -c "import yaml" 2> /dev/null; then
        good_python=python
    else
        good_python=python2
@ -2517,9 +2518,16 @@ __install_pip_pkgs() {

    # Install pip and pip dependencies
    if ! __check_command_exists "${_pip_cmd} --version"; then
-        __PACKAGES="${_py_pkg}-setuptools ${_py_pkg}-pip gcc ${_py_pkg}-devel"
+        __PACKAGES="${_py_pkg}-setuptools ${_py_pkg}-pip gcc"
        # shellcheck disable=SC2086
-        __yum_install_noinput ${__PACKAGES} || return 1
+        if [ "$DISTRO_NAME_L" = "debian" ];then
+            __PACKAGES="${__PACKAGES} ${_py_pkg}-dev"
+            __apt_get_install_noinput ${__PACKAGES} || return 1
+        else
+            __PACKAGES="${__PACKAGES} ${_py_pkg}-devel"
+            __yum_install_noinput ${__PACKAGES} || return 1
+        fi
+
    fi

    echoinfo "Installing pip packages: ${_pip_pkgs} using ${_py_exe}"
@ -2527,6 +2535,18 @@ __install_pip_pkgs() {
    ${_pip_cmd} install ${_pip_pkgs} || return 1
 }

+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  __install_tornado_pip
+#    PARAMETERS:  python executable
+#   DESCRIPTION:  Return 0 or 1 if successfully able to install tornado<5.0
+#----------------------------------------------------------------------------------------------------------------------
+__install_tornado_pip() {
+    # OS needs tornado <5.0 from pip
+    __check_pip_allowed "You need to allow pip based installations (-P) for Tornado <5.0 in order to install Salt on Python 3"
+    ## install pip if its not installed and install tornado
+    __install_pip_pkgs "tornado<5.0" "${1}" || return 1
+}
+
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
 #          NAME:  __install_pip_deps
 #   DESCRIPTION:  Return 0 or 1 if successfully able to install pip packages via requirements file
@ -3059,17 +3079,8 @@ install_ubuntu_check_services() {
 #   Debian Install Functions
 #
 __install_saltstack_debian_repository() {
-    if [ "$DISTRO_MAJOR_VERSION" -eq 10 ]; then
-        # Packages for Debian 10 at repo.saltstack.com are not yet available
-        # Set up repository for Debian 9 for Debian 10 for now until support
-        # is available at repo.saltstack.com for Debian 10.
-        echowarn "Debian 10 distribution detected, but stable packages requested. Trying packages from Debian 9. You may experience problems."
-        DEBIAN_RELEASE="9"
-        DEBIAN_CODENAME="stretch"
-    else
-        DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION"
-        DEBIAN_CODENAME="$DISTRO_CODENAME"
-    fi
+    DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION"
+    DEBIAN_CODENAME="$DISTRO_CODENAME"

    __PY_VERSION_REPO="apt"
    if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
@ -3150,6 +3161,24 @@ install_debian_deps() {
    return 0
 }

+install_debian_git_pre() {
+    if ! __check_command_exists git; then
+        __apt_get_install_noinput git || return 1
+    fi
+
+    if [ "$_INSECURE_DL" -eq $BS_FALSE ] && [ "${_SALT_REPO_URL%%://*}" = "https" ]; then
+        __apt_get_install_noinput ca-certificates
+    fi
+
+    __git_clone_and_checkout || return 1
+
+    # Let's trigger config_salt()
+    if [ "$_TEMP_CONFIG_DIR" = "null" ]; then
+        _TEMP_CONFIG_DIR="${_SALT_GIT_CHECKOUT_DIR}/conf/"
+        CONFIG_SALT_FUNC="config_salt"
+    fi
+}
+
 install_debian_git_deps() {
    if ! __check_command_exists git; then
        __apt_get_install_noinput git || return 1
@ -3299,7 +3328,25 @@ install_debian_9_git_deps() {
 }

 install_debian_10_git_deps() {
-    install_debian_9_git_deps || return 1
+    install_debian_git_pre || return 1
+
+    if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
+        _py=${_PY_EXE}
+        PY_PKG_VER=3
+        __PACKAGES="python${PY_PKG_VER}-distutils"
+    else
+        _py="python"
+        PY_PKG_VER=""
+        __PACKAGES=""
+    fi
+
+    __install_tornado_pip ${_py}|| return 1
+    __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-jinja2"
+    __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-tornado python${PY_PKG_VER}-yaml python${PY_PKG_VER}-zmq"
+
+    # shellcheck disable=SC2086
+    __apt_get_install_noinput ${__PACKAGES} || return 1
+
    return 0
 }

@ -3751,13 +3798,33 @@ install_centos_stable_deps() {
        __install_saltstack_rhel_repository || return 1
    fi

-    __PACKAGES="yum-utils chkconfig"
-
-    # YAML module is used for generating custom master/minion configs
-    if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
-        __PACKAGES="${__PACKAGES} python34-PyYAML"
+    if [ "$DISTRO_MAJOR_VERSION" -ge 8 ]; then
+        __PACKAGES="dnf-utils chkconfig"
    else
-        __PACKAGES="${__PACKAGES} PyYAML"
+        __PACKAGES="yum-utils chkconfig"
+    fi
+
+    if [ "$DISTRO_MAJOR_VERSION" -ge 8 ]; then
+        # YAML module is used for generating custom master/minion configs
+        if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
+            __PACKAGES="${__PACKAGES} python3-pyyaml"
+        else
+            __PACKAGES="${__PACKAGES} python2-pyyaml"
+        fi
+    elif [ "$DISTRO_MAJOR_VERSION" -eq 7 ]; then
+        # YAML module is used for generating custom master/minion configs
+        if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
+            __PACKAGES="${__PACKAGES} python36-PyYAML"
+        else
+            __PACKAGES="${__PACKAGES} PyYAML"
+        fi
+    else
+        # YAML module is used for generating custom master/minion configs
+        if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
+            __PACKAGES="${__PACKAGES} python34-PyYAML"
+        else
+            __PACKAGES="${__PACKAGES} PyYAML"
+        fi
    fi

    # shellcheck disable=SC2086
@ -3839,12 +3906,26 @@ install_centos_git_deps() {

    __git_clone_and_checkout || return 1

-    __PACKAGES="m2crypto"

+    __PACKAGES=""
+    _install_m2crypto_req=false
    if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then
-        # Packages are named python34-<whatever>
-        PY_PKG_VER=34
+        _py=${_PY_EXE}
+        if [ "$DISTRO_MAJOR_VERSION" -gt 6 ]; then
+            _install_m2crypto_req=true
+        fi
+        if [ "$DISTRO_MAJOR_VERSION" -ge 8 ]; then
+            # Packages are named python3-<whatever>
+            PY_PKG_VER=3
+        else
+            # Packages are named python36-<whatever>
+            PY_PKG_VER=36
+        fi
    else
+        if [ "$DISTRO_MAJOR_VERSION" -eq 6 ]; then
+            _install_m2crypto_req=true
+        fi
+        _py="python"
        PY_PKG_VER=""

        # Only Py2 needs python-futures
@ -3856,7 +3937,14 @@ install_centos_git_deps() {
        fi
    fi

-    __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2"
+    if [ "$DISTRO_MAJOR_VERSION" -ge 8 ]; then
+        __install_tornado_pip ${_py} || return 1
+        __PACKAGES="${__PACKAGES} python3-m2crypto"
+    else
+        __PACKAGES="${__PACKAGES} m2crypto python${PY_PKG_VER}-crypto"
+    fi
+
+    __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-jinja2"
    __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests"
    __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-tornado python${PY_PKG_VER}-zmq"

@ -3874,7 +3962,7 @@ install_centos_git_deps() {
        _PIP_PACKAGES="m2crypto!=0.33.0 jinja2 msgpack-python pycrypto PyYAML tornado<5.0 zmq futures>=2.0"

        # install swig and openssl on cent6
-        if [ "$DISTRO_MAJOR_VERSION" -eq 6 ]; then
+        if $_install_m2crypto_req; then
            __yum_install_noinput openssl-devel swig || return 1
        fi