From 182ef75d195f5c6cbee76f8420b0731bca54c502 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 25 Apr 2018 16:08:49 -0400 Subject: [PATCH 01/65] Add sum to README for 2018.04.25 release --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index ae5ca3f..ca6882b 100644 --- a/README.rst +++ b/README.rst @@ -28,6 +28,7 @@ sum** of the downloaded ``bootstrap-salt.sh`` file. The SHA256 sum of the ``bootstrap-salt.sh`` file, per release, is: +- 2018.04.25: ``e2e3397d6642ba6462174b4723f1b30d04229b75efc099a553e15ea727877dfb`` - 2017.12.13: ``c127b3aa4a8422f6b81f5b4a40d31d13cec97bf3a39bca9c11a28f24910a6895`` - 2017.08.17: ``909b4d35696b9867b34b22ef4b60edbc5a0e9f8d1ed8d05f922acb79a02e46e3`` - 2017.05.24: ``8c42c2e5ad3d4384ddc557da5c214ba3e40c056ca1b758d14a392c1364650e89`` From 03273de164cacb6ce2441cf2b592f0b205662f54 Mon Sep 17 00:00:00 2001 From: pjcreath Date: Wed, 14 Mar 2018 16:25:19 -0400 Subject: [PATCH 02/65] bootstrap-salt.sh: fix for silently ignored version argument on CentOS/RHEL This commit adds a warning when a pre-existing saltstack.repo on CentOS/RHEL causes the version argument to be ignored. It also allows the -F (forced overwrite) option to override this situation, overwriting the saltstack.repo file, and thus installing the specified version. Resolves #1209. --- bootstrap-salt.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 056a256..d3951b9 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3562,7 +3562,7 @@ __install_saltstack_rhel_repository() { gpg_key="SALTSTACK-GPG-KEY.pub" repo_file="/etc/yum.repos.d/saltstack.repo" - if [ ! -s "$repo_file" ]; then + if [ ! -s "$repo_file" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then cat <<_eof > "$repo_file" [saltstack] name=SaltStack ${repo_rev} Release Channel for RHEL/CentOS \$releasever @@ -3576,6 +3576,9 @@ _eof fetch_url="${HTTP_VAL}://${_REPO_URL}/yum/redhat/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${repo_rev}/" __rpm_import_gpg "${fetch_url}${gpg_key}" || return 1 + elif [ "$repo_rev" != "latest" ]; then + echowarn "saltstack.repo already exists, ignoring salt version argument." + echowarn "Use -F (forced overwrite) to install $repo_rev." fi return 0 From 056f8d8a3998651a9e2fa1ace81748499f8a97eb Mon Sep 17 00:00:00 2001 From: pjcreath Date: Fri, 16 Mar 2018 20:10:23 -0400 Subject: [PATCH 03/65] Add 'yum clean metadata' after updating the saltstack.repo file as requested. According to https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/sec-working_with_yum_cache this option "eliminates all files that yum uses to determine the remote availability of packages." --- bootstrap-salt.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d3951b9..04f8caf 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3576,6 +3576,7 @@ _eof fetch_url="${HTTP_VAL}://${_REPO_URL}/yum/redhat/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${repo_rev}/" __rpm_import_gpg "${fetch_url}${gpg_key}" || return 1 + yum clean metadata || return 1 elif [ "$repo_rev" != "latest" ]; then echowarn "saltstack.repo already exists, ignoring salt version argument." echowarn "Use -F (forced overwrite) to install $repo_rev." From 22a7bc373a298ab42d1fdbd0595369794aa336ed Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 10 May 2018 15:56:48 -0400 Subject: [PATCH 04/65] Add Ubuntu 18.04 support Fixes #1225 --- README.rst | 2 +- bootstrap-salt.sh | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index ca6882b..04658ea 100644 --- a/README.rst +++ b/README.rst @@ -276,7 +276,7 @@ Ubuntu and derivatives - KDE neon (based on Ubuntu 16.04) - Linux Mint 17/18 -- Ubuntu 14.04/16.04 and subsequent non-TLS releases (see below) +- Ubuntu 14.04/16.04/18.04 and subsequent non-LTS releases (see below) Ubuntu Best Effort Support: Non-LTS Releases ******************************************** diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 04f8caf..578a136 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -1373,6 +1373,9 @@ __ubuntu_codename_translation() { "17") DISTRO_CODENAME="artful" ;; + "18") + DISTRO_CODENAME="bionic" + ;; *) DISTRO_CODENAME="trusty" ;; @@ -2568,13 +2571,17 @@ __enable_universe_repository() { __install_saltstack_ubuntu_repository() { # Workaround for latest non-LTS ubuntu - if [ "$DISTRO_MAJOR_VERSION" -gt 16 ]; then - echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages from latest LTS release. You may experience problems." + if [ "$DISTRO_MAJOR_VERSION" -eq 17 ]; then + echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages for 16.04 LTS release. You may experience problems." UBUNTU_VERSION=16.04 UBUNTU_CODENAME="xenial" + elif [ "$DISTRO_MAJOR_VERSION" -gt 18 ]; then + echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages for latest LTS release. You may experience problems." + UBUNTU_VERSION=18.04 + UBUNTU_CODENAME="bionic" else - UBUNTU_VERSION=$DISTRO_VERSION - UBUNTU_CODENAME=$DISTRO_CODENAME + UBUNTU_VERSION=${DISTRO_VERSION} + UBUNTU_CODENAME=${DISTRO_CODENAME} fi __PACKAGES='' From 6d595996766841766b4a0e0559bfb49231cf73ad Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 10 May 2018 16:16:17 -0400 Subject: [PATCH 05/65] Fix the variable ref from #803 The version of shellcheck that was previously running on bootstrap PRs was out of date and not catching the fact that the VIRTUAL_ENV variable didn't exist. That variable should be _VIRTUALENV_DIR instead. --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 04f8caf..44e981b 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2394,7 +2394,7 @@ __create_virtualenv() { __activate_virtualenv() { set +o nounset # Is virtualenv empty - if [ -z "$VIRTUAL_ENV" ]; then + if [ -z "$_VIRTUALENV_DIR" ]; then __create_virtualenv || return 1 # shellcheck source=/dev/null . "${_VIRTUALENV_DIR}/bin/activate" || return 1 From c4eefe1b1b55e734b63caaa63569220054ee9c80 Mon Sep 17 00:00:00 2001 From: Jan Heidbrink Date: Mon, 7 May 2018 14:39:30 +0200 Subject: [PATCH 06/65] Check out repo even with NO_DEPS option This addresses issue 1217. --- bootstrap-salt.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 44e981b..f1375dc 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -6416,6 +6416,7 @@ daemons_running() { #====================================================================================================================== # Let's get the dependencies install function +DEP_FUNC_NAMES="" if [ ${_NO_DEPS} -eq $BS_FALSE ]; then DEP_FUNC_NAMES="install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}_${ITYPE}_deps" DEP_FUNC_NAMES="$DEP_FUNC_NAMES install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}_${ITYPE}_deps" @@ -6423,10 +6424,6 @@ if [ ${_NO_DEPS} -eq $BS_FALSE ]; then DEP_FUNC_NAMES="$DEP_FUNC_NAMES install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}_deps" DEP_FUNC_NAMES="$DEP_FUNC_NAMES install_${DISTRO_NAME_L}_${ITYPE}_deps" DEP_FUNC_NAMES="$DEP_FUNC_NAMES install_${DISTRO_NAME_L}_deps" -elif [ "${ITYPE}" = "git" ]; then - DEP_FUNC_NAMES="__git_clone_and_checkout" -else - DEP_FUNC_NAMES="" fi DEPS_INSTALL_FUNC="null" @@ -6571,6 +6568,7 @@ if [ "$INSTALL_FUNC" = "null" ]; then exit 1 fi + # Install dependencies if [ ${_NO_DEPS} -eq $BS_FALSE ] && [ $_CONFIG_ONLY -eq $BS_FALSE ]; then # Only execute function is not in config mode only @@ -6582,6 +6580,15 @@ if [ ${_NO_DEPS} -eq $BS_FALSE ] && [ $_CONFIG_ONLY -eq $BS_FALSE ]; then fi fi + +if [ "${ITYPE}" = "git" ] && [ ${_NO_DEPS} -eq ${BS_TRUE} ]; then + if ! __git_clone_and_checkout; then + echo "Failed to clone and checkout git repository." + exit 1 + fi +fi + + # Triggering config_salt() if overwriting master or minion configs if [ "$_CUSTOM_MASTER_CONFIG" != "null" ] || [ "$_CUSTOM_MINION_CONFIG" != "null" ]; then if [ "$_TEMP_CONFIG_DIR" = "null" ]; then From 04afc0dec365c618b53f6a558435af1f6ef69150 Mon Sep 17 00:00:00 2001 From: William Giokas <1007380@gmail.com> Date: Mon, 21 May 2018 12:55:26 -0600 Subject: [PATCH 07/65] Add Jenkinsfile Need to add a pipeline to use this. --- Jenkinsfile | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..be664af --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,21 @@ +def notifySuccessful(String stageName) { + slackSend (color: '#00FF00', message: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" + "\n Stage -- " + stageName) +} + +def notifyFailed(String stageName) { + slackSend (color: '#FF0000', message: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" + "\n Stage -- " + stageName) +} + +pipeline { + agent any + + stages { + stage('shellcheck') { + steps { + sh 'shellcheck -s sh -f checkstyle bootstrap-salt.sh | tee checkstyle.xml' + checkstyle pattern: '**/checkstyle.xml' + archiveArtifacts artifacts: '**/checkstyle.xml' + } + } + } +} From da7c173349619839330467a14e3c93c4be5b484a Mon Sep 17 00:00:00 2001 From: William Giokas <1007380@gmail.com> Date: Mon, 21 May 2018 15:34:22 -0600 Subject: [PATCH 08/65] Make this run on the new bootstrap instance --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index be664af..511ed50 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,7 +7,7 @@ def notifyFailed(String stageName) { } pipeline { - agent any + agent { label: 'bootstrap' } stages { stage('shellcheck') { From bec0bbb5d52465fb1256a2f6aac1fab2bebb5587 Mon Sep 17 00:00:00 2001 From: William Giokas <1007380@gmail.com> Date: Mon, 21 May 2018 15:47:06 -0600 Subject: [PATCH 09/65] Remove the colon --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 511ed50..b2bacdb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,7 +7,7 @@ def notifyFailed(String stageName) { } pipeline { - agent { label: 'bootstrap' } + agent { label 'bootstrap' } stages { stage('shellcheck') { From 46ca7143fa3220560e12fc60cb8b0479b85b9a3a Mon Sep 17 00:00:00 2001 From: William Giokas <1007380@gmail.com> Date: Tue, 22 May 2018 16:17:10 -0600 Subject: [PATCH 10/65] Update to scripting, different pr tests --- Jenkinsfile | 112 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 101 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b2bacdb..492c0e2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,21 +1,111 @@ +import java.util.Random + +Random rand = new Random() + +// ONLY CHANGE THIS BIT PLEASE +def distros = ["debian8", + "suse", + "centos6", + "arch", + "ubuntu-14.04", + "ubuntu-18.04", + "windows", + "ubuntu-16.04", + "centos7", + ] +def versions = ["stable", "git", "stable-old"] + +def prDistros = ["ubuntu-16.04", "centos7"] + distros[rand.nextInt(distros.size() - 2)] + +def prVersions = ["stable", "git"] + +// You probably shouldn't be messing with this stuff down here + def notifySuccessful(String stageName) { - slackSend (color: '#00FF00', message: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" + "\n Stage -- " + stageName) + slackSend (color: '#00FF00', message: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" + "\n Stage -- " + stageName) } def notifyFailed(String stageName) { - slackSend (color: '#FF0000', message: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" + "\n Stage -- " + stageName) + slackSend (color: '#FF0000', message: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" + "\n Stage -- " + stageName) } -pipeline { - agent { label 'bootstrap' } +def runKitchen(String distro) { + echo "kitchen create ${distro}" + echo "kitchen converge ${distro}" + echo "kitchen destroy ${distro}" +} - stages { - stage('shellcheck') { - steps { - sh 'shellcheck -s sh -f checkstyle bootstrap-salt.sh | tee checkstyle.xml' - checkstyle pattern: '**/checkstyle.xml' - archiveArtifacts artifacts: '**/checkstyle.xml' - } +def distroversions = [] +for (d in distros) { + for (v in versions) { + distroversions = distroversions + ["${d}-${v}"] + } +} + +def prDistroversions = [] +for (d in prDistros) { + for (v in prVersions) { + prDistroversions = prDistroversions + ["${d}-${v}"] + } +} + +def makeSetupRuns(dv) { + return { + node { + runKitchen("${dv}") } } } + +def setupRuns = distroversions.collectEntries { + ["kitchen-${it}" : makeSetupRuns("${it}")] +} + +def prSetupRuns = prDistroversions.collectEntries { + ["kitchen-${it}" : makeSetupRuns("${it}")] +} + +node ('bootstrap') { + stage('checkout') { checkout scm } + stage("var checking") { + echo "${env.CHANGE_ID}" + echo "${env.CHANGE_URL}" + } + stage('shellcheck') { + sh 'shellcheck -s sh -f checkstyle bootstrap-salt.sh | tee checkstyle.xml' + checkstyle pattern: '**/checkstyle.xml' + archiveArtifacts artifacts: '**/checkstyle.xml' + } + if (env.CHANGE_ID) { + // Running for a PR only runs against 4 random distros from a shorter list + stage('kitchen-pr') { + parallel prSetupRuns + } + } else { + // If we're not running for a pr we run *everything* + stage('kitchen-all') { + parallel setupRuns + } + } +} + +/* + * TODO: + * 1. Tests for each supported distro in bootstrap + branch shellcheck test (Shellcheck should be done) + * 2. Each distro needs a "stable" install (installs stable packages from our repo) and a "git" install (installs off of a git tag) + * 3. Running against each branch (stable, develop) + * 4. And probably a small subset against each pull request (similar to what we do in salt) + * + * Distros to check: + * Debian 8 + * Suse 42.1 + * CentOS 7 + * CentOS 6 + * Arch + * Ubuntu 16.04 + * Ubuntu 14.04 + * Windows + * + * Runs each against develop and stable + * + */ From b8ee1aa28e28cfc54093501cad91209728f6fa85 Mon Sep 17 00:00:00 2001 From: William Giokas <1007380@gmail.com> Date: Wed, 23 May 2018 10:36:37 -0600 Subject: [PATCH 11/65] Some cleanups to jenkins I removed the var checking, some trailing spaces --- Jenkinsfile | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 492c0e2..9be5c9b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -67,10 +67,6 @@ def prSetupRuns = prDistroversions.collectEntries { node ('bootstrap') { stage('checkout') { checkout scm } - stage("var checking") { - echo "${env.CHANGE_ID}" - echo "${env.CHANGE_URL}" - } stage('shellcheck') { sh 'shellcheck -s sh -f checkstyle bootstrap-salt.sh | tee checkstyle.xml' checkstyle pattern: '**/checkstyle.xml' @@ -95,7 +91,7 @@ node ('bootstrap') { * 2. Each distro needs a "stable" install (installs stable packages from our repo) and a "git" install (installs off of a git tag) * 3. Running against each branch (stable, develop) * 4. And probably a small subset against each pull request (similar to what we do in salt) - * + * * Distros to check: * Debian 8 * Suse 42.1 @@ -105,7 +101,7 @@ node ('bootstrap') { * Ubuntu 16.04 * Ubuntu 14.04 * Windows - * + * * Runs each against develop and stable - * + * */ From f0574fb2c409d4326959b09c5a8314b87969f70c Mon Sep 17 00:00:00 2001 From: William Giokas <1007380@gmail.com> Date: Wed, 23 May 2018 11:06:46 -0600 Subject: [PATCH 12/65] Update the distro list If you really need to test it against a different distro, you can go to the `Replay` thing in Jenkins and modify the basePRDistros or baseDistros list. --- Jenkinsfile | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9be5c9b..ce08a27 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,24 +3,25 @@ import java.util.Random Random rand = new Random() // ONLY CHANGE THIS BIT PLEASE -def distros = ["debian8", - "suse", - "centos6", - "arch", - "ubuntu-14.04", - "ubuntu-18.04", - "windows", - "ubuntu-16.04", - "centos7", - ] +def baseDistros = ["debian8", + "suse", + "centos6", + "arch", + "ubuntu-14.04", + "ubuntu-18.04", + "windows", + ] def versions = ["stable", "git", "stable-old"] -def prDistros = ["ubuntu-16.04", "centos7"] + distros[rand.nextInt(distros.size() - 2)] +def basePrDistros = ["ubuntu-16.04", + "centos7"] def prVersions = ["stable", "git"] // You probably shouldn't be messing with this stuff down here +def distros = (baseDistros + basePrDistros).unique() + def notifySuccessful(String stageName) { slackSend (color: '#00FF00', message: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" + "\n Stage -- " + stageName) } @@ -42,6 +43,8 @@ for (d in distros) { } } +def prDistros = basePrDistros + distros[rand.nextInt(baseDistros.size())] + def prDistroversions = [] for (d in prDistros) { for (v in prVersions) { From b4678508216baa52b09f16d8bdddb63a8a88c0f1 Mon Sep 17 00:00:00 2001 From: William Giokas <1007380@gmail.com> Date: Wed, 23 May 2018 11:10:55 -0600 Subject: [PATCH 13/65] Make sure we're not running a test twice --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ce08a27..3cde9f0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -43,7 +43,7 @@ for (d in distros) { } } -def prDistros = basePrDistros + distros[rand.nextInt(baseDistros.size())] +def prDistros = (basePrDistros + distros[rand.nextInt(baseDistros.size())]).unique() def prDistroversions = [] for (d in prDistros) { From e20866453d459b9b76966787d1414f6ee3366313 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 31 May 2018 10:13:12 -0400 Subject: [PATCH 14/65] Add Python3 package support for Ubuntu 16.04 Refs #1127 --- README.rst | 2 ++ bootstrap-salt.sh | 68 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 54 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index ca6882b..3d68837 100644 --- a/README.rst +++ b/README.rst @@ -277,6 +277,8 @@ Ubuntu and derivatives - KDE neon (based on Ubuntu 16.04) - Linux Mint 17/18 - Ubuntu 14.04/16.04 and subsequent non-TLS releases (see below) +- Python3 (see ``-x`` option): + - Ubuntu 16.04 Ubuntu Best Effort Support: Non-LTS Releases ******************************************** diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 04f8caf..c78ebd6 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -351,7 +351,7 @@ __usage() { -i Pass the salt-minion id. This will be stored under \${BS_SALT_ETC_DIR}/minion_id -p Extra-package to install while installing Salt dependencies. One package - per -p flag. You're responsible for providing the proper package name. + per -p flag. You are responsible for providing the proper package name. -H Use the specified HTTP proxy for all download URLs (including https://). For example: http://myproxy.example.com:3128 -b Assume that dependencies are already installed and software sources are @@ -385,11 +385,11 @@ __usage() { no ".bak" file will be created as either of those options will force a complete overwrite of the file. -q Quiet salt installation from git (setup.py install -q) - -x Changes the python version used to install a git version of salt. Currently - this is considered experimental and has only been tested on Centos 6. This - only works for git installations. + -x Changes the Python version used to install Salt. Currently, this is only + supported on Ubuntu 16 and CentOS 6. The CentOS 6 option only works with + git installations. -y Installs a different python version on host. Currently this has only been - tested with Centos 6 and is considered experimental. This will install the + tested with CentOS 6 and is considered experimental. This will install the ius repo on the box if disable repo is false. This must be used in conjunction with -x . For example: sh bootstrap.sh -P -y -x python2.7 git v2017.7.2 @@ -657,6 +657,22 @@ if [ "$_CUSTOM_MINION_CONFIG" != "null" ]; then fi fi +# Check if we're installing via a different Python executable and set major version variables +if [ -n "$_PY_EXE" ]; then + _PY_PKG_VER=$(echo "$_PY_EXE" | sed -r "s/\\.//g") + + _PY_MAJOR_VERSION=$(echo "$_PY_PKG_VER" | cut -c 7) + if [ "$_PY_MAJOR_VERSION" != 3 ] && [ "$_PY_MAJOR_VERSION" != 2 ]; then + echoerror "Detected -x option, but Python major version is not 2 or 3." + echoerror "The -x option must be passed as python2, python27, or python2.7 (or use the Python '3' versions of examples)." + exit 1 + fi + + echoinfo "Detected -x option. Using $_PY_EXE to install Salt." +else + _PY_PKG_VER="" +fi + # If the configuration directory or archive does not exist, error out if [ "$_TEMP_CONFIG_DIR" != "null" ]; then _TEMP_CONFIG_DIR="$(__check_config_dir "$_TEMP_CONFIG_DIR")" @@ -1102,9 +1118,7 @@ __install_python() { exit 1 fi - PY_PKG_V=$(echo "$_PY_EXE" | sed -r "s/\.//g") - __PACKAGES="${PY_PKG_V}" - + __PACKAGES="$_PY_PKG_VER" if [ ${_DISABLE_REPOS} -eq ${BS_FALSE} ]; then echoinfo "Attempting to install a repo to help provide a separate python package" @@ -2594,8 +2608,13 @@ __install_saltstack_ubuntu_repository() { # shellcheck disable=SC2086,SC2090 __apt_get_install_noinput ${__PACKAGES} || return 1 + __PY_VERSION_REPO="apt" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + __PY_VERSION_REPO="py3" + fi + # SaltStack's stable Ubuntu repository: - SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/apt/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/${STABLE_REV}" + SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${__PY_VERSION_REPO}/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/${STABLE_REV}" echo "deb $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/saltstack.list __apt_key_fetch "$SALTSTACK_UBUNTU_URL/SALTSTACK-GPG-KEY.pub" || return 1 @@ -2624,7 +2643,7 @@ install_ubuntu_deps() { __PACKAGES="upstart" fi - if [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then + if [ "$DISTRO_MAJOR_VERSION" -ge 16 ] && [ -z "$_PY_EXE" ]; then __PACKAGES="${__PACKAGES} python2.7" fi @@ -2740,12 +2759,23 @@ install_ubuntu_git_deps() { else install_ubuntu_stable_deps || return 1 - __PACKAGES="${__PACKAGES} python-crypto python-jinja2 python-m2crypto python-msgpack" - __PACKAGES="${__PACKAGES} python-requests python-tornado python-yaml python-zmq" + if [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + PY_PKG_VER=3 + else + PY_PKG_VER="" + + # There is no m2crypto package for Py3 at this time - only install for Py2 + __PACKAGES="${__PACKAGES} python-m2crypto" + fi + + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-tornado python${PY_PKG_VER}-yaml" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-zmq" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then # Install python-libcloud if asked to - __PACKAGES="${__PACKAGES} python-libcloud" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-libcloud" fi # shellcheck disable=SC2086 @@ -2795,10 +2825,16 @@ install_ubuntu_git() { __activate_virtualenv || return 1 fi - if [ -f "${_SALT_GIT_CHECKOUT_DIR}/salt/syspaths.py" ]; then - python setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR}" ${SETUP_PY_INSTALL_ARGS} install --install-layout=deb || return 1 + if [ -n "$_PY_EXE" ]; then + _PYEXE=${_PY_EXE} else - python setup.py ${SETUP_PY_INSTALL_ARGS} install --install-layout=deb || return 1 + _PYEXE=python2.7 + fi + + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/salt/syspaths.py" ]; then + ${_PYEXE} setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR}" ${SETUP_PY_INSTALL_ARGS} install --install-layout=deb || return 1 + else + ${_PYEXE} setup.py ${SETUP_PY_INSTALL_ARGS} install --install-layout=deb || return 1 fi return 0 From c136ddfdb9d2082be548f2d60d7894d8ace16873 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 31 May 2018 10:27:33 -0400 Subject: [PATCH 15/65] Fix rst formatting for nested bullet list --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 3d68837..0d24e08 100644 --- a/README.rst +++ b/README.rst @@ -278,6 +278,7 @@ Ubuntu and derivatives - Linux Mint 17/18 - Ubuntu 14.04/16.04 and subsequent non-TLS releases (see below) - Python3 (see ``-x`` option): + - Ubuntu 16.04 Ubuntu Best Effort Support: Non-LTS Releases From 27d3d9c3e28481a05b9d12ec7269c1cc1ffd5d54 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 4 Jun 2018 11:18:38 -0400 Subject: [PATCH 16/65] Add separate section to README for Python 3 support --- README.rst | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 0d24e08..03a82d1 100644 --- a/README.rst +++ b/README.rst @@ -277,9 +277,6 @@ Ubuntu and derivatives - KDE neon (based on Ubuntu 16.04) - Linux Mint 17/18 - Ubuntu 14.04/16.04 and subsequent non-TLS releases (see below) -- Python3 (see ``-x`` option): - - - Ubuntu 16.04 Ubuntu Best Effort Support: Non-LTS Releases ******************************************** @@ -313,8 +310,8 @@ UNIX systems - SmartOS -Unsupported Distro ------------------- +Unsupported Distributions +------------------------- If you are running a Linux distribution that is not supported yet or is not correctly identified, please run the following commands and report their output when creating an issue: @@ -327,6 +324,22 @@ please run the following commands and report their output when creating an issue For information on how to add support for a currently unsupported distribution, please refer to the `Contributing Guidelines`_. +Python 3 Support +---------------- + +Some distributions support installing Salt to use Python 3 instead of Python 2. The availability of +this offering, while limited, is as follows: + +- Ubuntu 16.04 + +Installing the Python 3 packages for Salt is done via the ``-x`` option: + +.. code:: console + + sh bootstrap-salt.sh -x python3 + +See the ``-x`` option for more information. + Testing ------- From 2000661cc64ecf6ed665b1701d9477472c196452 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 31 May 2018 13:07:35 -0400 Subject: [PATCH 17/65] Change "egrep" uses to be "grep -E" Fixes the following shellcheck violation: SC2196: egrep is non-standard and deprecated. Use grep -E instead. --- bootstrap-salt.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 6392860..b5c77fb 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -567,7 +567,7 @@ if [ "$#" -gt 0 ];then fi # Check installation type -if [ "$(echo "$ITYPE" | egrep '(stable|testing|daily|git)')" = "" ]; then +if [ "$(echo "$ITYPE" | grep -E '(stable|testing|daily|git)')" = "" ]; then echoerror "Installation type \"$ITYPE\" is not known..." exit 1 fi @@ -589,10 +589,10 @@ elif [ "$ITYPE" = "stable" ]; then if [ "$#" -eq 0 ];then STABLE_REV="latest" else - if [ "$(echo "$1" | egrep '^(latest|1\.6|1\.7|2014\.1|2014\.7|2015\.5|2015\.8|2016\.3|2016\.11|2017\.7|2018\.3)$')" != "" ]; then + if [ "$(echo "$1" | grep -E '^(latest|1\.6|1\.7|2014\.1|2014\.7|2015\.5|2015\.8|2016\.3|2016\.11|2017\.7|2018\.3)$')" != "" ]; then STABLE_REV="$1" shift - elif [ "$(echo "$1" | egrep '^([0-9]*\.[0-9]*\.[0-9]*)$')" != "" ]; then + elif [ "$(echo "$1" | grep -E '^([0-9]*\.[0-9]*\.[0-9]*)$')" != "" ]; then STABLE_REV="archive/$1" shift else @@ -923,7 +923,7 @@ __sort_release_files() { secondary_release_files="" # Sort know VS un-known files first for release_file in $(echo "${@}" | sed -r 's:[[:space:]]:\n:g' | sort -f | uniq); do - match=$(echo "$release_file" | egrep -i "${KNOWN_RELEASE_FILES}") + match=$(echo "$release_file" | grep -E -i "${KNOWN_RELEASE_FILES}") if [ "${match}" != "" ]; then primary_release_files="${primary_release_files} ${release_file}" else @@ -1031,11 +1031,11 @@ __gather_linux_system_info() { v=$(__parse_version_string "$rv") case $shortname in redhat ) - if [ "$(egrep 'CentOS' /etc/${rsource})" != "" ]; then + if [ "$(grep -E 'CentOS' /etc/${rsource})" != "" ]; then n="CentOS" - elif [ "$(egrep 'Scientific' /etc/${rsource})" != "" ]; then + elif [ "$(grep -E 'Scientific' /etc/${rsource})" != "" ]; then n="Scientific Linux" - elif [ "$(egrep 'Red Hat Enterprise Linux' /etc/${rsource})" != "" ]; then + elif [ "$(grep -E 'Red Hat Enterprise Linux' /etc/${rsource})" != "" ]; then n="ed at nterprise inux" else n="ed at inux" @@ -1272,7 +1272,7 @@ __ubuntu_derivatives_translation() { neon_16_ubuntu_base="16.04" # Translate Ubuntu derivatives to their base Ubuntu version - match=$(echo "$DISTRO_NAME_L" | egrep ${UBUNTU_DERIVATIVES}) + match=$(echo "$DISTRO_NAME_L" | grep -E ${UBUNTU_DERIVATIVES}) if [ "${match}" != "" ]; then case $match in @@ -1417,7 +1417,7 @@ __debian_derivatives_translation() { raspbian_9_debian_base="9.0" # Translate Debian derivatives to their base Debian version - match=$(echo "$DISTRO_NAME_L" | egrep ${DEBIAN_DERIVATIVES}) + match=$(echo "$DISTRO_NAME_L" | grep -E ${DEBIAN_DERIVATIVES}) if [ "${match}" != "" ]; then case $match in @@ -1728,14 +1728,14 @@ fi if ([ "${DISTRO_NAME_L}" != "ubuntu" ] && [ "$ITYPE" = "daily" ]); then echoerror "${DISTRO_NAME} does not have daily packages support" exit 1 -elif ([ "$(echo "${DISTRO_NAME_L}" | egrep '(debian|ubuntu|centos|red_hat|oracle|scientific|amazon)')" = "" ] && [ "$ITYPE" = "stable" ] && [ "$STABLE_REV" != "latest" ]); then +elif ([ "$(echo "${DISTRO_NAME_L}" | grep -E '(debian|ubuntu|centos|red_hat|oracle|scientific|amazon)')" = "" ] && [ "$ITYPE" = "stable" ] && [ "$STABLE_REV" != "latest" ]); then echoerror "${DISTRO_NAME} does not have major version pegged packages support" exit 1 fi # Only RedHat based distros have testing support if [ "${ITYPE}" = "testing" ]; then - if [ "$(echo "${DISTRO_NAME_L}" | egrep '(centos|red_hat|amazon|oracle)')" = "" ]; then + if [ "$(echo "${DISTRO_NAME_L}" | grep -E '(centos|red_hat|amazon|oracle)')" = "" ]; then echoerror "${DISTRO_NAME} does not have testing packages support" exit 1 fi @@ -4474,13 +4474,13 @@ install_amazon_linux_ami_deps() { repo_rev="latest" fi - if echo $repo_rev | egrep -q '^archive'; then + if echo $repo_rev | grep -E -q '^archive'; then year=$(echo "$repo_rev" | cut -d '/' -f 2 | cut -c1-4) else year=$(echo "$repo_rev" | cut -c1-4) fi - if echo "$repo_rev" | egrep -q '^(latest|2016\.11)$' || \ + if echo "$repo_rev" | grep -E -q '^(latest|2016\.11)$' || \ [ "$year" -gt 2016 ]; then _USEAWS=$BS_TRUE pkg_append="python27" From 765944555cb02584e1f78b37d6d0cf8b36cb7553 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 31 May 2018 13:28:18 -0400 Subject: [PATCH 18/65] Use explicit escaping with double backslashes Fixes the shellcheck errors that look like this: SC1117: Backslash is literal in "\n". Prefer explicit escaping: "\\n". --- bootstrap-salt.sh | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index b5c77fb..c81573a 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -61,11 +61,11 @@ __DEFAULT_SLEEP=3 _COLORS=${BS_COLORS:-$(tput colors 2>/dev/null || echo 0)} __detect_color_support() { if [ $? -eq 0 ] && [ "$_COLORS" -gt 2 ]; then - RC="\033[1;31m" - GC="\033[1;32m" - BC="\033[1;34m" - YC="\033[1;33m" - EC="\033[0m" + RC="\\033[1;31m" + GC="\\033[1;32m" + BC="\\033[1;34m" + YC="\\033[1;33m" + EC="\\033[0m" else RC="" GC="" @@ -82,7 +82,7 @@ __detect_color_support # DESCRIPTION: Echo errors to stderr. #---------------------------------------------------------------------------------------------------------------------- echoerror() { - printf "${RC} * ERROR${EC}: %s\n" "$@" 1>&2; + printf "${RC} * ERROR${EC}: %s\\n" "$@" 1>&2; } #--- FUNCTION ------------------------------------------------------------------------------------------------------- @@ -90,7 +90,7 @@ echoerror() { # DESCRIPTION: Echo information to stdout. #---------------------------------------------------------------------------------------------------------------------- echoinfo() { - printf "${GC} * INFO${EC}: %s\n" "$@"; + printf "${GC} * INFO${EC}: %s\\n" "$@"; } #--- FUNCTION ------------------------------------------------------------------------------------------------------- @@ -98,7 +98,7 @@ echoinfo() { # DESCRIPTION: Echo warning information to stdout. #---------------------------------------------------------------------------------------------------------------------- echowarn() { - printf "${YC} * WARN${EC}: %s\n" "$@"; + printf "${YC} * WARN${EC}: %s\\n" "$@"; } #--- FUNCTION ------------------------------------------------------------------------------------------------------- @@ -107,7 +107,7 @@ echowarn() { #---------------------------------------------------------------------------------------------------------------------- echodebug() { if [ "$_ECHO_DEBUG" -eq $BS_TRUE ]; then - printf "${BC} * DEBUG${EC}: %s\n" "$@"; + printf "${BC} * DEBUG${EC}: %s\\n" "$@"; fi } @@ -891,6 +891,7 @@ __derive_debian_numeric_version() { # DESCRIPTION: Strip single or double quotes from the provided string. #---------------------------------------------------------------------------------------------------------------------- __unquote_string() { + # shellcheck disable=SC1117 echo "$*" | sed -e "s/^\([\"\']\)\(.*\)\1\$/\2/g" } @@ -935,14 +936,14 @@ __sort_release_files() { max_prio="redhat-release centos-release oracle-release fedora-release" for entry in $max_prio; do if [ "$(echo "${primary_release_files}" | grep "$entry")" != "" ]; then - primary_release_files=$(echo "${primary_release_files}" | sed -e "s:\(.*\)\($entry\)\(.*\):\2 \1 \3:g") + primary_release_files=$(echo "${primary_release_files}" | sed -e "s:\\(.*\\)\\($entry\\)\\(.*\\):\\2 \\1 \\3:g") fi done # Now, least important goes last in the min_prio list min_prio="lsb-release" for entry in $min_prio; do if [ "$(echo "${primary_release_files}" | grep "$entry")" != "" ]; then - primary_release_files=$(echo "${primary_release_files}" | sed -e "s:\(.*\)\($entry\)\(.*\):\1 \3 \2:g") + primary_release_files=$(echo "${primary_release_files}" | sed -e "s:\\(.*\\)\\($entry\\)\\(.*\\):\\1 \\3 \\2:g") fi done @@ -1159,17 +1160,17 @@ __gather_sunos_system_info() { case "$line" in *OpenIndiana*oi_[0-9]*) DISTRO_NAME="OpenIndiana" - DISTRO_VERSION=$(echo "$line" | sed -nr "s/OpenIndiana(.*)oi_([[:digit:]]+)(.*)/\2/p") + DISTRO_VERSION=$(echo "$line" | sed -nr "s/OpenIndiana(.*)oi_([[:digit:]]+)(.*)/\\2/p") break ;; *OpenSolaris*snv_[0-9]*) DISTRO_NAME="OpenSolaris" - DISTRO_VERSION=$(echo "$line" | sed -nr "s/OpenSolaris(.*)snv_([[:digit:]]+)(.*)/\2/p") + DISTRO_VERSION=$(echo "$line" | sed -nr "s/OpenSolaris(.*)snv_([[:digit:]]+)(.*)/\\2/p") break ;; *Oracle*Solaris*[0-9]*) DISTRO_NAME="Oracle Solaris" - DISTRO_VERSION=$(echo "$line" | sed -nr "s/(Oracle Solaris) ([[:digit:]]+)(.*)/\2/p") + DISTRO_VERSION=$(echo "$line" | sed -nr "s/(Oracle Solaris) ([[:digit:]]+)(.*)/\\2/p") break ;; *Solaris*) @@ -2204,7 +2205,7 @@ __overwriteconfig() { fi # Convert json string to a yaml string and write it to config file. Output is dumped into tempfile. - $good_python -c "import json; import yaml; jsn=json.loads('$json'); yml=yaml.safe_dump(jsn, line_break='\n', default_flow_style=False); config_file=open('$target', 'w'); config_file.write(yml); config_file.close();" 2>$tempfile + "$good_python" -c "import json; import yaml; jsn=json.loads('$json'); yml=yaml.safe_dump(jsn, line_break='\\n', default_flow_style=False); config_file=open('$target', 'w'); config_file.write(yml); config_file.close();" 2>$tempfile # No python errors output to the tempfile if [ ! -s "$tempfile" ]; then @@ -2294,7 +2295,7 @@ __check_services_sysvinit() { servicename=$1 echodebug "Checking if service ${servicename} is enabled" - if [ "$(LC_ALL=C /sbin/chkconfig --list | grep "\<${servicename}\>" | grep '[2-5]:on')" != "" ]; then + if [ "$(LC_ALL=C /sbin/chkconfig --list | grep "\\<${servicename}\\>" | grep '[2-5]:on')" != "" ]; then echodebug "Service ${servicename} is enabled" return 0 else @@ -2374,7 +2375,7 @@ __check_services_alpine() { echodebug "Checking if service ${servicename} is enabled" # shellcheck disable=SC2086,SC2046,SC2144 - if rc-status $(rc-status -r) | tail -n +2 | grep -q "\<$servicename\>"; then + if rc-status $(rc-status -r) | tail -n +2 | grep -q "\\<$servicename\\>"; then echodebug "Service ${servicename} is enabled" return 0 else @@ -2428,7 +2429,7 @@ __activate_virtualenv() { __install_pip_pkgs() { _pip_pkgs="$1" _py_exe="$2" - _py_pkg=$(echo "$_py_exe" | sed -r "s/\.//g") + _py_pkg=$(echo "$_py_exe" | sed -r "s/\\.//g") _pip_cmd="${_py_exe} -m pip" if [ "${_py_exe}" = "" ]; then @@ -3570,7 +3571,7 @@ __install_epel_repository() { fi # Check if epel repo is already enabled and flag it accordingly - yum repolist | grep -q "^[!]\?${_EPEL_REPO}/" + yum repolist | grep -q "^[!]\\?${_EPEL_REPO}/" if [ $? -eq 0 ]; then _EPEL_REPOS_INSTALLED=$BS_TRUE return 0 @@ -5004,7 +5005,7 @@ install_freebsd_git_deps() { if [ ! -f salt/syspaths.py ]; then # We still can't provide the system paths, salt 0.16.x # Let's patch salt's source and adapt paths to what's expected on FreeBSD - echodebug "Replacing occurrences of '/etc/salt' with \'${_SALT_ETC_DIR}\'" + echodebug "Replacing occurrences of '/etc/salt' with ${_SALT_ETC_DIR}" # The list of files was taken from Salt's BSD port Makefile for file in conf/minion conf/master salt/config.py salt/client.py \ salt/modules/mysql.py salt/utils/parsers.py salt/modules/tls.py \ From 1a2220983526268bbdc4ebad568cfe13cf315c1b Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 31 May 2018 13:35:19 -0400 Subject: [PATCH 19/65] Fix final double quoting error Fixes the following shellcheck error: SC2086: Double quote to prevent globbing and word splitting. --- bootstrap-salt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index c81573a..982e21a 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -6656,8 +6656,8 @@ fi # Drop the master address if passed if [ "$_SALT_MASTER_ADDRESS" != "null" ]; then [ ! -d "$_SALT_ETC_DIR/minion.d" ] && mkdir -p "$_SALT_ETC_DIR/minion.d" - cat <<_eof > $_SALT_ETC_DIR/minion.d/99-master-address.conf -master: $_SALT_MASTER_ADDRESS + cat <<_eof > "$_SALT_ETC_DIR/minion.d/99-master-address.conf +master: $_SALT_MASTER_ADDRESS" _eof fi From 18e1b973f24ffd975184bec03d1c957f339de1a7 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 31 May 2018 13:36:26 -0400 Subject: [PATCH 20/65] Remove "Z" option from list of possible options The functionality for the "Z" option was removed a while ago, but the letter was never removed from the opt list. This change fixes the following shellcheck warning: SC2213: getopts specified -Z, but it's not handled by this 'case'. --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 982e21a..d146b88 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -400,7 +400,7 @@ EOT } # ---------- end of function __usage ---------- -while getopts ':hvnDc:g:Gyx:wk:s:MSNXCPFUKIA:i:Lp:dH:ZbflV:J:j:rR:aq' opt +while getopts ':hvnDc:g:Gyx:wk:s:MSNXCPFUKIA:i:Lp:dH:bflV:J:j:rR:aq' opt do case "${opt}" in From 65263f176ccd21b9011990a0af336c036bb2c437 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 4 Jun 2018 11:51:53 -0400 Subject: [PATCH 21/65] Shellcheck updates based on review --- bootstrap-salt.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d146b88..eb2ee5d 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -61,11 +61,11 @@ __DEFAULT_SLEEP=3 _COLORS=${BS_COLORS:-$(tput colors 2>/dev/null || echo 0)} __detect_color_support() { if [ $? -eq 0 ] && [ "$_COLORS" -gt 2 ]; then - RC="\\033[1;31m" - GC="\\033[1;32m" - BC="\\033[1;34m" - YC="\\033[1;33m" - EC="\\033[0m" + RC='\033[1;31m' + GC='\033[1;32m' + BC='\033[1;34m' + YC='\033[1;33m' + EC='\033[0m' else RC="" GC="" @@ -6656,8 +6656,8 @@ fi # Drop the master address if passed if [ "$_SALT_MASTER_ADDRESS" != "null" ]; then [ ! -d "$_SALT_ETC_DIR/minion.d" ] && mkdir -p "$_SALT_ETC_DIR/minion.d" - cat <<_eof > "$_SALT_ETC_DIR/minion.d/99-master-address.conf -master: $_SALT_MASTER_ADDRESS" + cat <<_eof > "$_SALT_ETC_DIR/minion.d/99-master-address.conf" +master: $_SALT_MASTER_ADDRESS _eof fi From 58e0e51d4dc9cd8180284c80a9919081cde87bb0 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 5 Jun 2018 15:35:48 -0400 Subject: [PATCH 22/65] Add Python3 package support for CentOS 7 Refs #1127 --- bootstrap-salt.sh | 53 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index eb2ee5d..fa96eb8 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2760,7 +2760,7 @@ install_ubuntu_git_deps() { else install_ubuntu_stable_deps || return 1 - if [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then PY_PKG_VER=3 else PY_PKG_VER="" @@ -3593,9 +3593,14 @@ __install_saltstack_rhel_repository() { repo_rev="latest" fi + __PY_VERSION_REPO="yum" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + __PY_VERSION_REPO="py3" + fi + # Avoid using '$releasever' variable for yum. # Instead, this should work correctly on all RHEL variants. - base_url="${HTTP_VAL}://${_REPO_URL}/yum/redhat/${DISTRO_MAJOR_VERSION}/\$basearch/${repo_rev}/" + base_url="${HTTP_VAL}://${_REPO_URL}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/\$basearch/${repo_rev}/" gpg_key="SALTSTACK-GPG-KEY.pub" repo_file="/etc/yum.repos.d/saltstack.repo" @@ -3611,7 +3616,7 @@ enabled=1 enabled_metadata=1 _eof - fetch_url="${HTTP_VAL}://${_REPO_URL}/yum/redhat/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${repo_rev}/" + fetch_url="${HTTP_VAL}://${_REPO_URL}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${repo_rev}/" __rpm_import_gpg "${fetch_url}${gpg_key}" || return 1 yum clean metadata || return 1 elif [ "$repo_rev" != "latest" ]; then @@ -3627,7 +3632,13 @@ install_centos_stable_deps() { yum -y update || return 1 fi - if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then + if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ] || ( [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ] ); then + + if [ "$_DISABLE_REPOS" -eq "$BS_TRUE" ]; then + echowarn "Detected -r or -R option while installing Salt packages for Python 3." + echowarn "Python 3 packages for Salt require the EPEL repository to be installed." + fi + __install_epel_repository || return 1 __install_saltstack_rhel_repository || return 1 fi @@ -3639,8 +3650,14 @@ install_centos_stable_deps() { __install_saltstack_rhel_repository || return 1 fi + __PACKAGES="yum-utils chkconfig" + # YAML module is used for generating custom master/minion configs - __PACKAGES="yum-utils chkconfig PyYAML" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + __PACKAGES="${__PACKAGES} python34-PyYAML" + else + __PACKAGES="${__PACKAGES} PyYAML" + fi # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -3721,15 +3738,29 @@ install_centos_git_deps() { __git_clone_and_checkout || return 1 - __PACKAGES="m2crypto python-crypto python-futures python-jinja2 python-msgpack" - __PACKAGES="${__PACKAGES} python-requests python-tornado python-zmq" + __PACKAGES="m2crypto" - if [ "$DISTRO_MAJOR_VERSION" -ge 7 ]; then - __PACKAGES="${__PACKAGES} systemd-python" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + # Packages are named python34- + PY_PKG_VER=34 + else + PY_PKG_VER="" + + # Only Py2 needs python-futures + __PACKAGES="${__PACKAGES} python-futures" + + # There is no systemd-python3 package as of this writing + if [ "$DISTRO_MAJOR_VERSION" -ge 7 ]; then + __PACKAGES="${__PACKAGES} systemd-python" + fi fi + __PACKAGES="python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-tornado python${PY_PKG_VER}-zmq" + if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} python-libcloud" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-libcloud" fi if [ "${_INSTALL_PY}" -eq "${BS_TRUE}" ]; then @@ -3737,7 +3768,7 @@ install_centos_git_deps() { __install_python || return 1 fi - if [ "${_PY_EXE}" != "" ]; then + if [ "${_PY_EXE}" != "" ] && [ "$_PIP_ALLOWED" -eq "$BS_TRUE" ]; then # If "-x" is defined, install dependencies with pip based on the Python version given. _PIP_PACKAGES="m2crypto jinja2 msgpack-python pycrypto PyYAML tornado<5.0 zmq" From e672bdc1b9b231976fa97d0b7801fb8f5402abf4 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 5 Jun 2018 15:39:13 -0400 Subject: [PATCH 23/65] Update README with CentOS 7 Python 3 support --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 03a82d1..65b130d 100644 --- a/README.rst +++ b/README.rst @@ -330,6 +330,7 @@ Python 3 Support Some distributions support installing Salt to use Python 3 instead of Python 2. The availability of this offering, while limited, is as follows: +- CentOS 7 - Ubuntu 16.04 Installing the Python 3 packages for Salt is done via the ``-x`` option: From dfa4f4650346c2cf95598700cd2985c0ab67264c Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 6 Jun 2018 14:55:45 -0400 Subject: [PATCH 24/65] Simplify logic: fail if -r/R is passed with -x python3 --- bootstrap-salt.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index fa96eb8..51070fd 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3632,13 +3632,14 @@ install_centos_stable_deps() { yum -y update || return 1 fi - if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ] || ( [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ] ); then - - if [ "$_DISABLE_REPOS" -eq "$BS_TRUE" ]; then - echowarn "Detected -r or -R option while installing Salt packages for Python 3." - echowarn "Python 3 packages for Salt require the EPEL repository to be installed." - fi + if [ "$_DISABLE_REPOS" -eq "$BS_TRUE" ] && [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + echoerror "Detected -r or -R option while installing Salt packages for Python 3." + echoerror "Python 3 packages for Salt require the EPEL repository to be installed." + echoerror "The -r and -R options are incompatible with -x and Python 3 bootstrap installs." + return 1 + fi + if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ]; then __install_epel_repository || return 1 __install_saltstack_rhel_repository || return 1 fi From 1f547fede8e47656cbb177adaff63efb2cbec5f1 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 6 Jun 2018 17:02:27 -0400 Subject: [PATCH 25/65] Add Python3 package support for Debian 9 Refs #1127 --- bootstrap-salt.sh | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 51070fd..7f90195 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3016,6 +3016,11 @@ __install_saltstack_debian_repository() { DEBIAN_CODENAME="$DISTRO_CODENAME" fi + __PY_VERSION_REPO="apt" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + __PY_VERSION_REPO="py3" + fi + __PACKAGES='' # Install downloader backend for GPG keys fetching @@ -3034,7 +3039,7 @@ __install_saltstack_debian_repository() { __apt_get_install_noinput ${__PACKAGES} || return 1 # amd64 is just a part of repository URI, 32-bit pkgs are hosted under the same location - SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/apt/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/${STABLE_REV}" + SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/${STABLE_REV}" echo "deb $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/saltstack.list" __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALTSTACK-GPG-KEY.pub" || return 1 @@ -3064,11 +3069,17 @@ install_debian_deps() { __apt_get_upgrade_noinput || return 1 fi + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + PY_PKG_VER=3 + else + PY_PKG_VER="" + fi + # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 __PACKAGES='procps pciutils' # YAML module is used for generating custom master/minion configs - __PACKAGES="${__PACKAGES} python-yaml" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-yaml" # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3204,13 +3215,24 @@ install_debian_9_git_deps() { __git_clone_and_checkout || return 1 - __PACKAGES="libzmq5 lsb-release python-apt python-backports-abc python-crypto" - __PACKAGES="${__PACKAGES} python-jinja2 python-m2crypto python-msgpack python-requests" - __PACKAGES="${__PACKAGES} python-systemd python-tornado python-yaml python-zmq" + __PACKAGES="libzmq5 lsb-release" + + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + PY_PKG_VER=3 + else + PY_PKG_VER="" + + # These packages are PY2-ONLY + __PACKAGES="${__PACKAGES} python-backports-abc python-m2crypto" + fi + + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-apt python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests python${PY_PKG_VER}-systemd" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-tornado python${PY_PKG_VER}-yaml python${PY_PKG_VER}-zmq" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then # Install python-libcloud if asked to - __PACKAGES="${__PACKAGES} python-libcloud" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-libcloud" fi # shellcheck disable=SC2086 @@ -3268,10 +3290,16 @@ install_debian_9_stable() { } install_debian_git() { - if [ -f "${_SALT_GIT_CHECKOUT_DIR}/salt/syspaths.py" ]; then - python setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR}" ${SETUP_PY_INSTALL_ARGS} install --install-layout=deb || return 1 + if [ -n "$_PY_EXE" ]; then + _PYEXE=${_PY_EXE} else - python setup.py ${SETUP_PY_INSTALL_ARGS} install --install-layout=deb || return 1 + _PYEXE=python + fi + + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/salt/syspaths.py" ]; then + ${_PYEXE} setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR}" ${SETUP_PY_INSTALL_ARGS} install --install-layout=deb || return 1 + else + ${_PYEXE} setup.py ${SETUP_PY_INSTALL_ARGS} install --install-layout=deb || return 1 fi } From 747b58bdb1821e920beec0eaadf2e33a880de77b Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 6 Jun 2018 17:04:19 -0400 Subject: [PATCH 26/65] Update -x inline docs and README file with python3 support info --- README.rst | 1 + bootstrap-salt.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 65b130d..1ec1239 100644 --- a/README.rst +++ b/README.rst @@ -331,6 +331,7 @@ Some distributions support installing Salt to use Python 3 instead of Python 2. this offering, while limited, is as follows: - CentOS 7 +- Debian 9 - Ubuntu 16.04 Installing the Python 3 packages for Salt is done via the ``-x`` option: diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 7f90195..d59bb0e 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -386,8 +386,8 @@ __usage() { a complete overwrite of the file. -q Quiet salt installation from git (setup.py install -q) -x Changes the Python version used to install Salt. Currently, this is only - supported on Ubuntu 16 and CentOS 6. The CentOS 6 option only works with - git installations. + supported on CentOS 7, Debian 9, Ubuntu 16 and CentOS 6. The CentOS 6 + option only works with git installations. -y Installs a different python version on host. Currently this has only been tested with CentOS 6 and is considered experimental. This will install the ius repo on the box if disable repo is false. This must be used in conjunction From f08d0459168689de653b52e392635b0223dbd740 Mon Sep 17 00:00:00 2001 From: N Date: Fri, 8 Jun 2018 16:42:15 +0100 Subject: [PATCH 27/65] OpenSuse_Leap_15.0 repo support --- bootstrap-salt.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d59bb0e..93605fd 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -976,6 +976,7 @@ __gather_linux_system_info() { [ "$n" = "$DISTRO_NAME" ] && DISTRO_NAME="" || DISTRO_NAME="$n" elif [ "${DISTRO_NAME}" = "openSUSE project" ]; then # lsb_release -si returns "openSUSE project" on openSUSE 12.3 + # lsb_release -si returns "openSUSE" on openSUSE 15.n DISTRO_NAME="opensuse" elif [ "${DISTRO_NAME}" = "SUSE LINUX" ]; then if [ "$(lsb_release -sd | grep -i opensuse)" != "" ]; then @@ -1048,7 +1049,7 @@ __gather_linux_system_info() { debian ) n="Debian" ;; ubuntu ) n="Ubuntu" ;; fedora ) n="Fedora" ;; - suse ) n="SUSE" ;; + suse|opensuse ) n="SUSE" ;; mandrake*|mandriva ) n="Mandriva" ;; gentoo ) n="Gentoo" ;; slackware ) n="Slackware" ;; @@ -1090,7 +1091,7 @@ __gather_linux_system_info() { n="Debian" v=$(__derive_debian_numeric_version "$v") ;; - sles ) + sles|opensuse ) n="SUSE" v="${rv}" ;; @@ -5508,7 +5509,12 @@ __set_suse_pkg_repo() { elif [ "${DISTRO_MAJOR_VERSION}" -ge 42 ]; then DISTRO_REPO="openSUSE_Leap_${DISTRO_MAJOR_VERSION}.${DISTRO_MINOR_VERSION}" elif [ "${DISTRO_MAJOR_VERSION}" -lt 42 ]; then - DISTRO_REPO="SLE_${DISTRO_MAJOR_VERSION}_SP${SUSE_PATCHLEVEL}" + case ${DISTRO_MAJOR_VERSION} in + 15) DISTRO_REPO="openSUSE_Leap_${DISTRO_MAJOR_VERSION}.${DISTRO_MINOR_VERSION}" + ;; + *) DISTRO_REPO="SLE_${DISTRO_MAJOR_VERSION}_SP${SUSE_PATCHLEVEL}" + ;; + esac fi if [ "$_DOWNSTREAM_PKG_REPO" -eq $BS_TRUE ]; then From b3767e2bdc059328ff55c5a13c620ac92d6cace0 Mon Sep 17 00:00:00 2001 From: Ky-Anh Huynh Date: Tue, 19 Jun 2018 13:14:40 +0700 Subject: [PATCH 28/65] Disable shell error about unbound variable during daily install --- bootstrap-salt.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d59bb0e..4051aa3 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -584,6 +584,10 @@ if [ "$ITYPE" = "git" ]; then # Disable shell warning about unbound variable during git install STABLE_REV="latest" +elif [ "$ITYPE" = "daily" ]; then + # Disable shell error about unbound variable during daily install + STABLE_REV="latest" + # If doing stable install, check if version specified elif [ "$ITYPE" = "stable" ]; then if [ "$#" -eq 0 ];then From 584c97de5d2b64703cca7292dccc91a0728a6240 Mon Sep 17 00:00:00 2001 From: Ky-Anh Huynh Date: Tue, 19 Jun 2018 13:21:11 +0700 Subject: [PATCH 29/65] Return immediately if apt-get fails --- bootstrap-salt.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d59bb0e..7fc59b4 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2621,7 +2621,7 @@ __install_saltstack_ubuntu_repository() { __apt_key_fetch "$SALTSTACK_UBUNTU_URL/SALTSTACK-GPG-KEY.pub" || return 1 __wait_for_apt - apt-get update + apt-get update || return 1 } install_ubuntu_deps() { @@ -2634,7 +2634,7 @@ install_ubuntu_deps() { __enable_universe_repository || return 1 __wait_for_apt - apt-get update + apt-get update || return 1 fi __PACKAGES='' @@ -2691,12 +2691,12 @@ install_ubuntu_stable_deps() { export DEBIAN_FRONTEND=noninteractive __wait_for_apt - apt-get update + apt-get update || return 1 if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then if [ "${_INSECURE_DL}" -eq $BS_TRUE ]; then __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && - apt-key update && apt-get update + apt-key update && apt-get update || return 1 fi __apt_get_upgrade_noinput || return 1 @@ -2718,7 +2718,7 @@ install_ubuntu_daily_deps() { __enable_universe_repository || return 1 add-apt-repository -y ppa:saltstack/salt-daily || return 1 - apt-get update + apt-get update || return 1 fi if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then @@ -2730,7 +2730,7 @@ install_ubuntu_daily_deps() { install_ubuntu_git_deps() { __wait_for_apt - apt-get update + apt-get update || return 1 if ! __check_command_exists git; then __apt_get_install_noinput git-core || return 1 @@ -3045,7 +3045,7 @@ __install_saltstack_debian_repository() { __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALTSTACK-GPG-KEY.pub" || return 1 __wait_for_apt - apt-get update + apt-get update || return 1 } install_debian_deps() { @@ -3057,13 +3057,13 @@ install_debian_deps() { export DEBIAN_FRONTEND=noninteractive __wait_for_apt - apt-get update + apt-get update || return 1 if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then # Try to update GPG keys first if allowed if [ "${_INSECURE_DL}" -eq $BS_TRUE ]; then __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && - apt-key update && apt-get update + apt-key update && apt-get update || return 1 fi __apt_get_upgrade_noinput || return 1 From 8e037381402d4cb73af4055776a240a7c11d00d8 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 19 Jun 2018 11:44:44 -0400 Subject: [PATCH 30/65] Add python 3 support for Ubuntu 18 Stable installs work fine, but git installs need an additional package to be installed (python3-setuptools). Otherwise, the script fails with the following error: ``` * INFO: Running install_ubuntu_git() Traceback (most recent call last): File "setup.py", line 24, in import distutils.dist ModuleNotFoundError: No module named 'distutils.dist' ``` --- README.rst | 1 + bootstrap-salt.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 4d43092..1491cb2 100644 --- a/README.rst +++ b/README.rst @@ -333,6 +333,7 @@ this offering, while limited, is as follows: - CentOS 7 - Debian 9 - Ubuntu 16.04 +- Ubuntu 18.04 Installing the Python 3 packages for Salt is done via the ``-x`` option: diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 2e0789e..27d77b5 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2774,6 +2774,8 @@ install_ubuntu_git_deps() { if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then PY_PKG_VER=3 + + __PACKAGES="${__PACKAGES} python3-setuptools" else PY_PKG_VER="" From abb3d7455b6e9c4ae08a93529e3212321088fa84 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 19 Jun 2018 11:52:17 -0400 Subject: [PATCH 31/65] Add new contributors to the authors list --- AUTHORS.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index 02bd0e6..8159084 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -69,6 +69,7 @@ Guillaume Derval GuillaumeDerval guillaume@guillaumederval.be gweis gweis Henrik Holmboe holmboe Howard Mei HowardMei howardleomei@gmail.com +Jan Heidbrink jheidbrink James Booth absolutejam vvalentine1337@gmail.com Jared E Stroud jaredestroud jaredestroud@gmail.com Jasper Lievisse Adriaanse jasperla jasper@humppa.nl @@ -80,6 +81,7 @@ Justin Anderson justinta justin.ta@outlook.com Justin Findlay jfindlay jfindlay@gmail.com Karl Grzeszczak karlgrz Kenneth Wilke KennethWilke +Ky-Anh Huynh icy lomeroe lomeroe Liu Xiaohui oreh herolxh@gmail.com Lorenzo Perone lopezio lorenzo.perone@yellowspace.net @@ -101,6 +103,7 @@ Michael Scherer mscherer Michele Bologna mbologna michele.bologna@gmail.com Mike Carlson m87carlson mike@bayphoto.com Mike Place cachedout mp@saltstack.com +N noelmcloughlin noel.maclochlainn@gmail.com nasenbaer13 nasenbaer13 nevins-b nevins-b Nicole Thomas rallytime nicole@saltstack.com From 782a4b4ffe4b0de3ac9b9cd20c93edfe9fd08dbc Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 19 Jun 2018 11:58:09 -0400 Subject: [PATCH 32/65] Update changelog with recent develop changes --- ChangeLog | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ChangeLog b/ChangeLog index a396058..faa0755 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,17 @@ +Version TBD (In Progress on the Develop Branch): + * Add python 3 support for Ubuntu 18 (rallytime) #1248 + * Return immediately if apt-get fails (icy) #1247 + * Disable shell error about unbound variable during daily install (icy) #1246 + * OpenSuse_Leap_15.0 saltstack repo support (noelmcloughlin) #1244 + * Add Python3 package support for Debian 9 (rallytime) #1243 + * Add Python3 package support for CentOS 7 (rallytime) #1242 + * Shellcheck fixes found when running latest version (rallytime) #1239 + * Add Python3 package support for Ubuntu 16.04 (rallytime) #1238 + * Fix the variable ref from #803 (rallytime) #1229 + * Add Ubuntu 18.04 support (rallytime) #1228 + * Call __git_clone_and_checkout directly (jheidbrink) #1226 + * Fix for silently ignored version argument on CentOS/RHEL (pjcreath) #1210 + Version 2018.04.25: * Install py-tornado4 for FreeBSD. (abednarik) #1219 * Remove COPR repos configuration for Fedora (vutny ) #1211 From 375b74a1ba7e5e17228cd27ebc2e3f9757cf7e75 Mon Sep 17 00:00:00 2001 From: Ky-Anh Huynh Date: Fri, 22 Jun 2018 10:19:41 +0700 Subject: [PATCH 33/65] Remove daily install option on Ubuntu system. Reference: * https://github.com/saltstack/salt-bootstrap/issues/1245 * https://github.com/saltstack/salt-pack/issues/562 --- bootstrap-salt.sh | 40 +++------------------------------ tests/bootstrap/test_install.py | 33 --------------------------- 2 files changed, 3 insertions(+), 70 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 27d77b5..0f72cd8 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -289,7 +289,6 @@ __usage() { for packages available at repo.saltstack.com - stable [version] Install a specific version. Only supported for packages available at repo.saltstack.com - - daily Ubuntu specific: configure SaltStack Daily PPA - testing RHEL-family specific: configure EPEL testing repo - git Install from the head of the develop branch - git [ref] Install from any git ref (such as a branch, tag, or @@ -300,7 +299,6 @@ __usage() { - ${__ScriptName} stable - ${__ScriptName} stable 2017.7 - ${__ScriptName} stable 2017.7.2 - - ${__ScriptName} daily - ${__ScriptName} testing - ${__ScriptName} git - ${__ScriptName} git 2017.7 @@ -567,7 +565,7 @@ if [ "$#" -gt 0 ];then fi # Check installation type -if [ "$(echo "$ITYPE" | grep -E '(stable|testing|daily|git)')" = "" ]; then +if [ "$(echo "$ITYPE" | grep -E '(stable|testing|git)')" = "" ]; then echoerror "Installation type \"$ITYPE\" is not known..." exit 1 fi @@ -584,10 +582,6 @@ if [ "$ITYPE" = "git" ]; then # Disable shell warning about unbound variable during git install STABLE_REV="latest" -elif [ "$ITYPE" = "daily" ]; then - # Disable shell error about unbound variable during daily install - STABLE_REV="latest" - # If doing stable install, check if version specified elif [ "$ITYPE" = "stable" ]; then if [ "$#" -eq 0 ];then @@ -980,7 +974,7 @@ __gather_linux_system_info() { [ "$n" = "$DISTRO_NAME" ] && DISTRO_NAME="" || DISTRO_NAME="$n" elif [ "${DISTRO_NAME}" = "openSUSE project" ]; then # lsb_release -si returns "openSUSE project" on openSUSE 12.3 - # lsb_release -si returns "openSUSE" on openSUSE 15.n + # lsb_release -si returns "openSUSE" on openSUSE 15.n DISTRO_NAME="opensuse" elif [ "${DISTRO_NAME}" = "SUSE LINUX" ]; then if [ "$(lsb_release -sd | grep -i opensuse)" != "" ]; then @@ -1733,11 +1727,7 @@ elif [ "${DISTRO_NAME_L}" = "debian" ]; then __debian_codename_translation fi -# Only Ubuntu has daily packages, let's let users know about that -if ([ "${DISTRO_NAME_L}" != "ubuntu" ] && [ "$ITYPE" = "daily" ]); then - echoerror "${DISTRO_NAME} does not have daily packages support" - exit 1 -elif ([ "$(echo "${DISTRO_NAME_L}" | grep -E '(debian|ubuntu|centos|red_hat|oracle|scientific|amazon)')" = "" ] && [ "$ITYPE" = "stable" ] && [ "$STABLE_REV" != "latest" ]); then +if ([ "$(echo "${DISTRO_NAME_L}" | grep -E '(debian|ubuntu|centos|red_hat|oracle|scientific|amazon)')" = "" ] && [ "$ITYPE" = "stable" ] && [ "$STABLE_REV" != "latest" ]); then echoerror "${DISTRO_NAME} does not have major version pegged packages support" exit 1 fi @@ -2722,24 +2712,6 @@ install_ubuntu_stable_deps() { install_ubuntu_deps || return 1 } -install_ubuntu_daily_deps() { - __wait_for_apt - install_ubuntu_stable_deps || return 1 - - if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then - __enable_universe_repository || return 1 - - add-apt-repository -y ppa:saltstack/salt-daily || return 1 - apt-get update || return 1 - fi - - if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then - __apt_get_upgrade_noinput || return 1 - fi - - return 0 -} - install_ubuntu_git_deps() { __wait_for_apt apt-get update || return 1 @@ -2828,12 +2800,6 @@ install_ubuntu_stable() { return 0 } -install_ubuntu_daily() { - install_ubuntu_stable || return 1 - - return 0 -} - install_ubuntu_git() { # Activate virtualenv before install if [ "${_VIRTUALENV_DIR}" != "null" ]; then diff --git a/tests/bootstrap/test_install.py b/tests/bootstrap/test_install.py index 5f52aa9..b06d4cf 100644 --- a/tests/bootstrap/test_install.py +++ b/tests/bootstrap/test_install.py @@ -270,39 +270,6 @@ class InstallationTestCase(BootstrapTestCase): ) ) - def test_install_daily(self): - args = [] - if requires_pip_based_installations(): - args.append('-P') - - args.append('daily') - - rc, out, err = self.run_script( - args=args, timeout=15 * 60, stream_stds=True - ) - if GRAINS['os'] in ('Ubuntu', 'Trisquel', 'Mint'): - self.assert_script_result( - 'Failed to install daily', - 0, (rc, out, err) - ) - - # Try to get the versions report - self.assert_script_result( - 'Failed to get the versions report (\'--versions-report\')', - 0, - self.run_script( - script=None, - args=('salt-minion', '--versions-report'), - timeout=15 * 60, - stream_stds=True - ) - ) - else: - self.assert_script_result( - 'Although system is not Ubuntu, we managed to install', - 1, (rc, out, err) - ) - def test_install_testing(self): args = [] if requires_pip_based_installations(): From 20a89d31bcb059584c4a935bd85e6182f8d31a1a Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 9 Jul 2018 11:13:56 -0400 Subject: [PATCH 34/65] Avoid subshell overhead and remove unnecessary parens around conditions Fixes the following shellcheck violations: - SC2233: Remove superfluous (..) around condition. - SC2235: Use { ..; } instead of (..) to avoid subshell overhead. --- bootstrap-salt.sh | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 0f72cd8..beb2612 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -1511,8 +1511,8 @@ __check_end_of_life_versions() { # = 17.04 if [ "$DISTRO_MAJOR_VERSION" -lt 14 ] || \ [ "$DISTRO_MAJOR_VERSION" -eq 15 ] || \ - ([ "$DISTRO_MAJOR_VERSION" -eq 17 ] && [ "$DISTRO_MINOR_VERSION" -eq 04 ]) || \ - ([ "$DISTRO_MAJOR_VERSION" -lt 17 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]); then + { [ "$DISTRO_MAJOR_VERSION" -eq 17 ] && [ "$DISTRO_MINOR_VERSION" -eq 04 ]; } || \ + { [ "$DISTRO_MAJOR_VERSION" -lt 17 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; }; then echoerror "End of life distributions are not supported." echoerror "Please consider upgrading to the next stable. See:" echoerror " https://wiki.ubuntu.com/Releases" @@ -1526,7 +1526,7 @@ __check_end_of_life_versions() { # <= 13.X # <= 42.1 if [ "$DISTRO_MAJOR_VERSION" -le 13 ] || \ - ([ "$DISTRO_MAJOR_VERSION" -eq 42 ] && [ "$DISTRO_MINOR_VERSION" -le 1 ]); then + { [ "$DISTRO_MAJOR_VERSION" -eq 42 ] && [ "$DISTRO_MINOR_VERSION" -le 1 ]; }; then echoerror "End of life distributions are not supported." echoerror "Please consider upgrading to the next stable. See:" echoerror " http://en.opensuse.org/Lifetime" @@ -1544,8 +1544,8 @@ __check_end_of_life_versions() { SUSE_PATCHLEVEL="00" fi if [ "$DISTRO_MAJOR_VERSION" -lt 11 ] || \ - ([ "$DISTRO_MAJOR_VERSION" -eq 11 ] && [ "$SUSE_PATCHLEVEL" -lt 04 ]) || \ - ([ "$DISTRO_MAJOR_VERSION" -eq 12 ] && [ "$SUSE_PATCHLEVEL" -lt 02 ]); then + { [ "$DISTRO_MAJOR_VERSION" -eq 11 ] && [ "$SUSE_PATCHLEVEL" -lt 04 ]; } || \ + { [ "$DISTRO_MAJOR_VERSION" -eq 12 ] && [ "$SUSE_PATCHLEVEL" -lt 02 ]; }; then echoerror "Versions lower than SuSE 11 SP4 or 12 SP2 are not supported." echoerror "Please consider upgrading to the next stable" echoerror " https://www.suse.com/lifecycle/" @@ -1625,7 +1625,8 @@ __check_end_of_life_versions() { freebsd) # FreeBSD versions lower than 9.1 are not supported. - if ([ "$DISTRO_MAJOR_VERSION" -eq 9 ] && [ "$DISTRO_MINOR_VERSION" -lt 01 ]) || [ "$DISTRO_MAJOR_VERSION" -lt 9 ]; then + if { [ "$DISTRO_MAJOR_VERSION" -eq 9 ] && [ "$DISTRO_MINOR_VERSION" -lt 01 ]; } || \ + [ "$DISTRO_MAJOR_VERSION" -lt 9 ]; then echoerror "Versions lower than FreeBSD 9.1 are not supported." exit 1 fi @@ -1727,7 +1728,7 @@ elif [ "${DISTRO_NAME_L}" = "debian" ]; then __debian_codename_translation fi -if ([ "$(echo "${DISTRO_NAME_L}" | grep -E '(debian|ubuntu|centos|red_hat|oracle|scientific|amazon)')" = "" ] && [ "$ITYPE" = "stable" ] && [ "$STABLE_REV" != "latest" ]); then +if [ "$(echo "${DISTRO_NAME_L}" | grep -E '(debian|ubuntu|centos|red_hat|oracle|scientific|amazon)')" = "" ] && [ "$ITYPE" = "stable" ] && [ "$STABLE_REV" != "latest" ]; then echoerror "${DISTRO_NAME} does not have major version pegged packages support" exit 1 fi @@ -1742,13 +1743,13 @@ if [ "${ITYPE}" = "testing" ]; then fi # Only Ubuntu has support for installing to virtualenvs -if ([ "${DISTRO_NAME_L}" != "ubuntu" ] && [ "$_VIRTUALENV_DIR" != "null" ]); then +if [ "${DISTRO_NAME_L}" != "ubuntu" ] && [ "$_VIRTUALENV_DIR" != "null" ]; then echoerror "${DISTRO_NAME} does not have -V support" exit 1 fi # Only Ubuntu has support for pip installing all packages -if ([ "${DISTRO_NAME_L}" != "ubuntu" ] && [ $_PIP_ALL -eq $BS_TRUE ]); then +if [ "${DISTRO_NAME_L}" != "ubuntu" ] && [ $_PIP_ALL -eq $BS_TRUE ]; then echoerror "${DISTRO_NAME} does not have -a support" exit 1 fi @@ -3310,7 +3311,7 @@ install_debian_git_post() { # Configure SystemD for Debian 8 "Jessie" and later if [ -f /bin/systemctl ]; then if [ ! -f /lib/systemd/system/salt-${fname}.service ] || \ - ([ -f /lib/systemd/system/salt-${fname}.service ] && [ $_FORCE_OVERWRITE -eq $BS_TRUE ]); then + { [ -f /lib/systemd/system/salt-${fname}.service ] && [ $_FORCE_OVERWRITE -eq $BS_TRUE ]; }; then if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" ]; then __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" /lib/systemd/system __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.environment" "/etc/default/salt-${fname}" @@ -3329,7 +3330,7 @@ install_debian_git_post() { # Install initscripts for Debian 7 "Wheezy" elif [ ! -f "/etc/init.d/salt-$fname" ] || \ - ([ -f "/etc/init.d/salt-$fname" ] && [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]); then + { [ -f "/etc/init.d/salt-$fname" ] && [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; }; then if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-$fname.init" ]; then __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init" "/etc/init.d/salt-${fname}" __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.environment" "/etc/default/salt-${fname}" @@ -3844,13 +3845,13 @@ install_centos_git_post() { if [ -f /bin/systemctl ]; then if [ ! -f "/usr/lib/systemd/system/salt-${fname}.service" ] || \ - ([ -f "/usr/lib/systemd/system/salt-${fname}.service" ] && [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]); then + { [ -f "/usr/lib/systemd/system/salt-${fname}.service" ] && [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; }; then __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/rpm/salt-${fname}.service" /usr/lib/systemd/system fi SYSTEMD_RELOAD=$BS_TRUE elif [ ! -f "/etc/init.d/salt-$fname" ] || \ - ([ -f "/etc/init.d/salt-$fname" ] && [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]); then + { [ -f "/etc/init.d/salt-$fname" ] && [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; }; then __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/rpm/salt-${fname}" /etc/init.d chmod +x /etc/init.d/salt-${fname} fi @@ -5699,7 +5700,8 @@ install_opensuse_git_post() { if [ -f /bin/systemctl ]; then use_usr_lib=$BS_FALSE - if [ "${DISTRO_MAJOR_VERSION}" -gt 13 ] || ([ "${DISTRO_MAJOR_VERSION}" -eq 13 ] && [ "${DISTRO_MINOR_VERSION}" -ge 2 ]); then + if [ "${DISTRO_MAJOR_VERSION}" -gt 13 ] || \ + { [ "${DISTRO_MAJOR_VERSION}" -eq 13 ] && [ "${DISTRO_MINOR_VERSION}" -ge 2 ]; }; then use_usr_lib=$BS_TRUE fi From 99fcb4dc26bb4580139bbbc066e6ec5d6a053bbf Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 9 Jul 2018 11:20:32 -0400 Subject: [PATCH 35/65] Use "command -v" instead of "which" command ``which`` is non-standard. Fixes the following shellcheck error: - SC2230: which is non-standard. Use builtin 'command -v' instead. --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index beb2612..8d72bba 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -5095,7 +5095,7 @@ install_freebsd_11_stable() { install_freebsd_git() { # /usr/local/bin/python2 in FreeBSD is a symlink to /usr/local/bin/python2.7 - __PYTHON_PATH=$(readlink -f "$(which python2)") + __PYTHON_PATH=$(readlink -f "$(command -v python2)") __ESCAPED_PYTHON_PATH=$(echo "${__PYTHON_PATH}" | sed 's/\//\\\//g') # Install from git From bd1d4ad8bdefc98f3125d1e4306b17ac7a42d675 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 9 Jul 2018 11:23:18 -0400 Subject: [PATCH 36/65] Fix wordsplitting error found by shellcheck SC2231: Quote expansions in this for loop glob to prevent wordsplitting, e.g. "$dir"/*.txt . --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 8d72bba..395ea9e 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -6440,7 +6440,7 @@ preseed_master() { SEED_DEST="$_PKI_DIR/master/minions" [ -d "$SEED_DEST" ] || (mkdir -p "$SEED_DEST" && chmod 700 "$SEED_DEST") || return 1 - for keyfile in $_TEMP_KEYS_DIR/*; do + for keyfile in "$_TEMP_KEYS_DIR"/*; do keyfile=$(basename "${keyfile}") src_keyfile="${_TEMP_KEYS_DIR}/${keyfile}" dst_keyfile="${SEED_DEST}/${keyfile}" From f45e965fa017e24bf19e54a9b5694e29ebad2455 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 9 Jul 2018 12:49:11 -0400 Subject: [PATCH 37/65] Check exitcodes directly, rather than indirectly This is a new requirement in recent versions of shellcheck. This commit fixes the following shellcheck error: - SC2181: Check exit code directly with e.g. 'if mycmd;', not indirectly with $?. --- bootstrap-salt.sh | 73 +++++++++++++++++------------------------------ 1 file changed, 26 insertions(+), 47 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 395ea9e..e85d522 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -43,10 +43,6 @@ __ScriptArgs="$*" #====================================================================================================================== -#====================================================================================================================== -# LET THE BLACK MAGIC BEGIN!!!! -#====================================================================================================================== - # Bootstrap script truth values BS_TRUE=1 BS_FALSE=0 @@ -60,6 +56,7 @@ __DEFAULT_SLEEP=3 #---------------------------------------------------------------------------------------------------------------------- _COLORS=${BS_COLORS:-$(tput colors 2>/dev/null || echo 0)} __detect_color_support() { + # shellcheck disable=SC2181 if [ $? -eq 0 ] && [ "$_COLORS" -gt 2 ]; then RC='\033[1;31m' GC='\033[1;32m' @@ -463,8 +460,7 @@ LOGPIPE="/tmp/$( echo "$__ScriptName" | sed s/.sh/.logpipe/g )" # Create our logging pipe # On FreeBSD we have to use mkfifo instead of mknod -mknod "$LOGPIPE" p >/dev/null 2>&1 || mkfifo "$LOGPIPE" >/dev/null 2>&1 -if [ $? -ne 0 ]; then +if ! (mknod "$LOGPIPE" p >/dev/null 2>&1 || mkfifo "$LOGPIPE" >/dev/null 2>&1); then echoerror "Failed to create the named pipe required to log" exit 1 fi @@ -789,8 +785,7 @@ __fetch_verify() { test "$(stat --format=%s "$fetch_verify_tmpf")" -eq "$fetch_verify_size" && \ test "$(md5sum "$fetch_verify_tmpf" | awk '{ print $1 }')" = "$fetch_verify_sum" && \ cat "$fetch_verify_tmpf" && \ - rm -f "$fetch_verify_tmpf" - if [ $? -eq 0 ]; then + if rm -f "$fetch_verify_tmpf"; then return 0 fi echo "Failed verification of $fetch_verify_url" @@ -960,6 +955,8 @@ __gather_linux_system_info() { # Let's test if the lsb_release binary is available rv=$(lsb_release >/dev/null 2>&1) + + # shellcheck disable=SC2181 if [ $? -eq 0 ]; then DISTRO_NAME=$(lsb_release -si) if [ "${DISTRO_NAME}" = "Scientific" ]; then @@ -1176,8 +1173,7 @@ __gather_sunos_system_info() { DISTRO_NAME="Solaris" # Let's make sure we not actually on a Joyent's SmartOS VM since some releases # don't have SmartOS in `/etc/release`, only `Solaris` - uname -v | grep joyent >/dev/null 2>&1 - if [ $? -eq 0 ]; then + if uname -v | grep joyent >/dev/null 2>&1; then DISTRO_NAME="SmartOS" fi break @@ -1944,8 +1940,7 @@ __git_clone_and_checkout() { # HEAD; instead it will simply reset to itself. Check the ref to see # if it is a branch name, check out the branch, and pull in the # changes. - git branch -a | grep -q "${GIT_REV}" - if [ $? -eq 0 ]; then + if git branch -a | grep -q "${GIT_REV}"; then echodebug "Rebasing the cloned repository branch" git pull --rebase || return 1 fi @@ -1968,8 +1963,7 @@ __git_clone_and_checkout() { if [ "$(git clone 2>&1 | grep 'single-branch')" != "" ]; then # The "--single-branch" option is supported, attempt shallow cloning echoinfo "Attempting to shallow clone $GIT_REV from Salt's repository ${_SALT_REPO_URL}" - git clone --depth 1 --branch "$GIT_REV" "$_SALT_REPO_URL" "$__SALT_CHECKOUT_REPONAME" - if [ $? -eq 0 ]; then + if git clone --depth 1 --branch "$GIT_REV" "$_SALT_REPO_URL" "$__SALT_CHECKOUT_REPONAME"; then # shellcheck disable=SC2164 cd "${_SALT_GIT_CHECKOUT_DIR}" __SHALLOW_CLONE=$BS_TRUE @@ -2266,9 +2260,7 @@ __check_services_upstart() { echodebug "Checking if service ${servicename} is enabled" # Check if service is enabled to start at boot - initctl list | grep "${servicename}" > /dev/null 2>&1 - - if [ $? -eq 0 ]; then + if initctl list | grep "${servicename}" > /dev/null 2>&1; then echodebug "Service ${servicename} is enabled" return 0 else @@ -2922,8 +2914,7 @@ install_ubuntu_restart_daemons() { if [ -f /bin/systemctl ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then echodebug "There's systemd support while checking salt-$fname" systemctl stop salt-$fname > /dev/null 2>&1 - systemctl start salt-$fname.service - [ $? -eq 0 ] && continue + systemctl start salt-$fname.service && continue # We failed to start the service, let's test the SysV code below echodebug "Failed to start salt-$fname using systemd" fi @@ -2931,13 +2922,11 @@ install_ubuntu_restart_daemons() { if [ -f /sbin/initctl ]; then echodebug "There's upstart support while checking salt-$fname" - status salt-$fname 2>/dev/null | grep -q running - if [ $? -eq 0 ]; then + if status salt-$fname 2>/dev/null | grep -q running; then stop salt-$fname || (echodebug "Failed to stop salt-$fname" && return 1) fi - start salt-$fname - [ $? -eq 0 ] && continue + start salt-$fname && continue # We failed to start the service, let's test the SysV code below echodebug "Failed to start salt-$fname using Upstart" fi @@ -3580,8 +3569,7 @@ __install_epel_repository() { fi # Check if epel repo is already enabled and flag it accordingly - yum repolist | grep -q "^[!]\\?${_EPEL_REPO}/" - if [ $? -eq 0 ]; then + if yum repolist | grep -q "^[!]\\?${_EPEL_REPO}/"; then _EPEL_REPOS_INSTALLED=$BS_TRUE return 0 fi @@ -3880,8 +3868,7 @@ install_centos_restart_daemons() { if [ -f /sbin/initctl ] && [ -f /etc/init/salt-${fname}.conf ]; then # We have upstart support and upstart knows about our service - /sbin/initctl status salt-$fname > /dev/null 2>&1 - if [ $? -ne 0 ]; then + if ! /sbin/initctl status salt-$fname > /dev/null 2>&1; then # Everything is in place and upstart gave us an error code? Fail! return 1 fi @@ -3889,9 +3876,8 @@ install_centos_restart_daemons() { # upstart knows about this service. # Let's try to stop it, and then start it /sbin/initctl stop salt-$fname > /dev/null 2>&1 - /sbin/initctl start salt-$fname > /dev/null 2>&1 # Restart service - if [ $? -ne 0 ]; then + if ! /sbin/initctl start salt-$fname > /dev/null 2>&1; then # Failed the restart?! return 1 fi @@ -4422,6 +4408,7 @@ install_alpine_linux_post() { script_url="${_SALTSTACK_REPO_URL%.git}/raw/develop/pkg/alpine/salt-$fname" [ -f "/etc/init.d/salt-$fname" ] || __fetch_url "/etc/init.d/salt-$fname" "$script_url" + # shellcheck disable=SC2181 if [ $? -eq 0 ]; then chmod +x "/etc/init.d/salt-$fname" else @@ -5564,6 +5551,7 @@ install_opensuse_stable_deps() { fi __zypper --gpg-auto-import-keys refresh + # shellcheck disable=SC2181 if [ $? -ne 0 ] && [ $? -ne 4 ]; then # If the exit code is not 0, and it's not 4 (failed to update a # repository) return a failure. Otherwise continue. @@ -6657,8 +6645,7 @@ fi if [ ${_NO_DEPS} -eq $BS_FALSE ] && [ $_CONFIG_ONLY -eq $BS_FALSE ]; then # Only execute function is not in config mode only echoinfo "Running ${DEPS_INSTALL_FUNC}()" - $DEPS_INSTALL_FUNC - if [ $? -ne 0 ]; then + if ! ${DEPS_INSTALL_FUNC}; then echoerror "Failed to run ${DEPS_INSTALL_FUNC}()!!!" exit 1 fi @@ -6682,8 +6669,7 @@ if [ "$_CUSTOM_MASTER_CONFIG" != "null" ] || [ "$_CUSTOM_MINION_CONFIG" != "null if [ ${_NO_DEPS} -eq $BS_FALSE ] && [ $_CONFIG_ONLY -eq $BS_TRUE ]; then # Execute function to satisfy dependencies for configuration step echoinfo "Running ${DEPS_INSTALL_FUNC}()" - $DEPS_INSTALL_FUNC - if [ $? -ne 0 ]; then + if ! ${DEPS_INSTALL_FUNC}; then echoerror "Failed to run ${DEPS_INSTALL_FUNC}()!!!" exit 1 fi @@ -6693,8 +6679,7 @@ fi # Configure Salt if [ "$CONFIG_SALT_FUNC" != "null" ] && [ "$_TEMP_CONFIG_DIR" != "null" ]; then echoinfo "Running ${CONFIG_SALT_FUNC}()" - $CONFIG_SALT_FUNC - if [ $? -ne 0 ]; then + if ! ${CONFIG_SALT_FUNC}; then echoerror "Failed to run ${CONFIG_SALT_FUNC}()!!!" exit 1 fi @@ -6717,8 +6702,7 @@ fi # Pre-seed master keys if [ "$PRESEED_MASTER_FUNC" != "null" ] && [ "$_TEMP_KEYS_DIR" != "null" ]; then echoinfo "Running ${PRESEED_MASTER_FUNC}()" - $PRESEED_MASTER_FUNC - if [ $? -ne 0 ]; then + if ! ${PRESEED_MASTER_FUNC}; then echoerror "Failed to run ${PRESEED_MASTER_FUNC}()!!!" exit 1 fi @@ -6728,8 +6712,7 @@ fi if [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then # Only execute function is not in config mode only echoinfo "Running ${INSTALL_FUNC}()" - $INSTALL_FUNC - if [ $? -ne 0 ]; then + if ! ${INSTALL_FUNC}; then echoerror "Failed to run ${INSTALL_FUNC}()!!!" exit 1 fi @@ -6738,8 +6721,7 @@ fi # Run any post install function. Only execute function if not in config mode only if [ "$POST_INSTALL_FUNC" != "null" ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then echoinfo "Running ${POST_INSTALL_FUNC}()" - $POST_INSTALL_FUNC - if [ $? -ne 0 ]; then + if ! ${POST_INSTALL_FUNC}; then echoerror "Failed to run ${POST_INSTALL_FUNC}()!!!" exit 1 fi @@ -6748,8 +6730,7 @@ fi # Run any check services function, Only execute function if not in config mode only if [ "$CHECK_SERVICES_FUNC" != "null" ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then echoinfo "Running ${CHECK_SERVICES_FUNC}()" - $CHECK_SERVICES_FUNC - if [ $? -ne 0 ]; then + if ! ${CHECK_SERVICES_FUNC}; then echoerror "Failed to run ${CHECK_SERVICES_FUNC}()!!!" exit 1 fi @@ -6760,8 +6741,7 @@ if [ "$STARTDAEMONS_INSTALL_FUNC" != "null" ] && [ ${_START_DAEMONS} -eq $BS_TRU echoinfo "Running ${STARTDAEMONS_INSTALL_FUNC}()" echodebug "Waiting ${_SLEEP} seconds for processes to settle before checking for them" sleep ${_SLEEP} - $STARTDAEMONS_INSTALL_FUNC - if [ $? -ne 0 ]; then + if ! ${STARTDAEMONS_INSTALL_FUNC}; then echoerror "Failed to run ${STARTDAEMONS_INSTALL_FUNC}()!!!" exit 1 fi @@ -6772,8 +6752,7 @@ if [ "$DAEMONS_RUNNING_FUNC" != "null" ] && [ ${_START_DAEMONS} -eq $BS_TRUE ]; echoinfo "Running ${DAEMONS_RUNNING_FUNC}()" echodebug "Waiting ${_SLEEP} seconds for processes to settle before checking for them" sleep ${_SLEEP} # Sleep a little bit to let daemons start - $DAEMONS_RUNNING_FUNC - if [ $? -ne 0 ]; then + if ! ${DAEMONS_RUNNING_FUNC}; then echoerror "Failed to run ${DAEMONS_RUNNING_FUNC}()!!!" for fname in api master minion syndic; do From befd25d429aa9e10aff6fe7f4dcfe09ec1363d06 Mon Sep 17 00:00:00 2001 From: N Date: Tue, 10 Jul 2018 17:03:27 +0100 Subject: [PATCH 38/65] fix for opensuse15 if no lsb_release pkg --- bootstrap-salt.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 0f72cd8..a489989 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -1089,10 +1089,14 @@ __gather_linux_system_info() { n="Debian" v=$(__derive_debian_numeric_version "$v") ;; - sles|opensuse ) + sles ) n="SUSE" v="${rv}" ;; + opensuse-leap ) + n="opensuse" + v="${rv}" + ;; * ) n=${nn} ;; From c8152fc3bfcfbc1a4d0cc4174397b81632107e09 Mon Sep 17 00:00:00 2001 From: N Date: Wed, 11 Jul 2018 00:07:01 +0100 Subject: [PATCH 39/65] delete-fifo before make-fifo --- bootstrap-salt.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index e85d522..122f4b8 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -457,6 +457,8 @@ shift $((OPTIND-1)) # Define our logging file and pipe paths LOGFILE="/tmp/$( echo "$__ScriptName" | sed s/.sh/.log/g )" LOGPIPE="/tmp/$( echo "$__ScriptName" | sed s/.sh/.logpipe/g )" +# Ensure no residual pipe exists +rm "$LOGPIPE" 2>/dev/null # Create our logging pipe # On FreeBSD we have to use mkfifo instead of mknod From 855512da4a6567db9a69d6a3ee40f57c1fe75a2a Mon Sep 17 00:00:00 2001 From: William Giokas <1007380@gmail.com> Date: Tue, 10 Jul 2018 18:29:39 -0600 Subject: [PATCH 40/65] Comment out all of the kitchen runs These aren't doing anything except taking up time on random hosts it seems. Better to remove them. --- Jenkinsfile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 492c0e2..fb36344 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -76,17 +76,17 @@ node ('bootstrap') { checkstyle pattern: '**/checkstyle.xml' archiveArtifacts artifacts: '**/checkstyle.xml' } - if (env.CHANGE_ID) { - // Running for a PR only runs against 4 random distros from a shorter list - stage('kitchen-pr') { - parallel prSetupRuns - } - } else { - // If we're not running for a pr we run *everything* - stage('kitchen-all') { - parallel setupRuns - } - } + // if (env.CHANGE_ID) { + // // Running for a PR only runs against 4 random distros from a shorter list + // stage('kitchen-pr') { + // parallel prSetupRuns + // } + // } else { + // // If we're not running for a pr we run *everything* + // stage('kitchen-all') { + // parallel setupRuns + // } + // } } /* From 576d2bbe3415f762dcc0b3d3403979d451cc23f3 Mon Sep 17 00:00:00 2001 From: William Giokas <1007380@gmail.com> Date: Tue, 10 Jul 2018 18:31:04 -0600 Subject: [PATCH 41/65] Use stack to get a newer shellcheck --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 492c0e2..c61cc1d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -72,7 +72,7 @@ node ('bootstrap') { echo "${env.CHANGE_URL}" } stage('shellcheck') { - sh 'shellcheck -s sh -f checkstyle bootstrap-salt.sh | tee checkstyle.xml' + sh 'stack exec -- shellcheck -s sh -f checkstyle bootstrap-salt.sh | tee checkstyle.xml' checkstyle pattern: '**/checkstyle.xml' archiveArtifacts artifacts: '**/checkstyle.xml' } From a491b71aab59993d67beb9a827a29fc67a3d7220 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 11 Jul 2018 10:29:02 -0400 Subject: [PATCH 42/65] Remove support for Ubutu 17.10 Ubuntu 17.10 will be EOL any day now, as Ubuntu's release calendar says support will end in July 2018. This commit removes support for the 17.10 (non-LTS) release. The README file was also updated to clarify when support for non-LTS releases will be removed from bootstrap. --- README.rst | 4 ++++ bootstrap-salt.sh | 15 ++++----------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 1491cb2..60c7bed 100644 --- a/README.rst +++ b/README.rst @@ -289,6 +289,9 @@ release instead. For example, when installing Salt on Ubuntu 17.10, the bootstrap script will setup the repository for Ubuntu 16.04 from `SaltStack's Ubuntu repository`_ and install the 16.04 packages. +Non-LTS Ubuntu releases are not supported once the release reaches End-of-Life as defined by +`Ubuntu's release schedule`_. + Other Linux distributions ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -401,6 +404,7 @@ Salt components, custom configurations, and even `pre-accepted Minion keys`_ alr .. _`SaltStack's corporate repository`: https://repo.saltstack.com/ .. _`SaltStack's Debian repository`: http://repo.saltstack.com/#debian .. _`SaltStack's Ubuntu repository`: http://repo.saltstack.com/#ubuntu +.. _`Ubuntu's release schedule`: https://wiki.ubuntu.com/Releases .. _Vagrant: http://www.vagrantup.com diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 122f4b8..1e70776 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -1382,9 +1382,6 @@ __ubuntu_codename_translation() { "16") DISTRO_CODENAME="xenial" ;; - "17") - DISTRO_CODENAME="artful" - ;; "18") DISTRO_CODENAME="bionic" ;; @@ -1506,11 +1503,11 @@ __check_end_of_life_versions() { # = 14.10 # = 15.04, 15.10 # = 16.10 - # = 17.04 + # = 17.04, 17.10 if [ "$DISTRO_MAJOR_VERSION" -lt 14 ] || \ [ "$DISTRO_MAJOR_VERSION" -eq 15 ] || \ - { [ "$DISTRO_MAJOR_VERSION" -eq 17 ] && [ "$DISTRO_MINOR_VERSION" -eq 04 ]; } || \ - { [ "$DISTRO_MAJOR_VERSION" -lt 17 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; }; then + [ "$DISTRO_MAJOR_VERSION" -eq 17 ] || \ + { [ "$DISTRO_MAJOR_VERSION" -eq 16 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; }; then echoerror "End of life distributions are not supported." echoerror "Please consider upgrading to the next stable. See:" echoerror " https://wiki.ubuntu.com/Releases" @@ -2576,11 +2573,7 @@ __enable_universe_repository() { __install_saltstack_ubuntu_repository() { # Workaround for latest non-LTS ubuntu - if [ "$DISTRO_MAJOR_VERSION" -eq 17 ]; then - echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages for 16.04 LTS release. You may experience problems." - UBUNTU_VERSION=16.04 - UBUNTU_CODENAME="xenial" - elif [ "$DISTRO_MAJOR_VERSION" -gt 18 ]; then + if [ "$DISTRO_MAJOR_VERSION" -gt 18 ]; then echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages for latest LTS release. You may experience problems." UBUNTU_VERSION=18.04 UBUNTU_CODENAME="bionic" From ba2ba1d1beea5a806f9f0656412401c9e1fcb353 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 11 Jul 2018 10:38:53 -0400 Subject: [PATCH 43/65] Remove support for Fedora 26 Fedora 26 is EOL as of the end of May, therefore the bootstrap script no longer needs to support it. --- README.rst | 2 +- bootstrap-salt.sh | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 1491cb2..0ae82ed 100644 --- a/README.rst +++ b/README.rst @@ -257,7 +257,7 @@ Red Hat family - Amazon Linux 2012.3 and later - CentOS 6/7 - Cloud Linux 6/7 -- Fedora 26/27/28 (install latest stable from standard repositories) +- Fedora 27/28 (install latest stable from standard repositories) - Oracle Linux 6/7 - Red Hat Enterprise Linux 6/7 - Scientific Linux 6/7 diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 122f4b8..91560bc 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -1552,8 +1552,8 @@ __check_end_of_life_versions() { ;; fedora) - # Fedora lower than 26 are no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 26 ]; then + # Fedora lower than 27 are no longer supported + if [ "$DISTRO_MAJOR_VERSION" -lt 27 ]; then echoerror "End of life distributions are not supported." echoerror "Please consider upgrading to the next stable. See:" echoerror " https://fedoraproject.org/wiki/Releases" @@ -3400,15 +3400,9 @@ install_debian_check_services() { install_fedora_deps() { - __PACKAGES="libyaml m2crypto PyYAML python-crypto python-jinja2" + __PACKAGES="dnf-utils libyaml m2crypto PyYAML python-crypto python-jinja2" __PACKAGES="${__PACKAGES} python2-msgpack python2-requests python-zmq" - if [ "$DISTRO_MAJOR_VERSION" -lt 26 ]; then - __PACKAGES="${__PACKAGES} yum-utils" - else - __PACKAGES="${__PACKAGES} dnf-utils" - fi - # shellcheck disable=SC2086 dnf install -y ${__PACKAGES} || return 1 From 14959e04bdf69d1ae97d70234cf411021dee0a0f Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 11 Jul 2018 11:11:58 -0400 Subject: [PATCH 44/65] Remove support for openSUSE Leap 42.2 Leap 42.2 is EOL and has been replaced by 42.3 This commit also simplifies some of the logic in checking for suse versions. openSUSE 13.X is no longer supported and hasn't been for some time, so we don't need to worry about checking for those versions any longer. The script will bail out at the `__check_end_of_life_versions` functions early on in the install process. --- README.rst | 2 +- bootstrap-salt.sh | 20 +++++++------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index 0ae82ed..638b3c8 100644 --- a/README.rst +++ b/README.rst @@ -266,7 +266,7 @@ Red Hat family SUSE family ~~~~~~~~~~~ -- openSUSE Leap 42.2/42.3 +- openSUSE Leap 42.3 - openSUSE Tumbleweed 2015 - SUSE Linux Enterprise Server 11 SP4, 12 SP2 diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 8dc6807..c50d89e 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -1526,9 +1526,9 @@ __check_end_of_life_versions() { # openSUSE versions not supported # # <= 13.X - # <= 42.1 - if [ "$DISTRO_MAJOR_VERSION" -le 13 ] || \ - { [ "$DISTRO_MAJOR_VERSION" -eq 42 ] && [ "$DISTRO_MINOR_VERSION" -le 1 ]; }; then + # <= 42.2 + if [ "$DISTRO_MAJOR_VERSION" -lt 15 ] || \ + { [ "$DISTRO_MAJOR_VERSION" -eq 42 ] && [ "$DISTRO_MINOR_VERSION" -le 2 ]; }; then echoerror "End of life distributions are not supported." echoerror "Please consider upgrading to the next stable. See:" echoerror " http://en.opensuse.org/Lifetime" @@ -5473,15 +5473,10 @@ __set_suse_pkg_repo() { # Set distro repo variable if [ "${DISTRO_MAJOR_VERSION}" -gt 2015 ]; then DISTRO_REPO="openSUSE_Tumbleweed" - elif [ "${DISTRO_MAJOR_VERSION}" -ge 42 ]; then + elif [ "${DISTRO_MAJOR_VERSION}" -ge 42 ] || [ "${DISTRO_MAJOR_VERSION}" -eq 15 ]; then DISTRO_REPO="openSUSE_Leap_${DISTRO_MAJOR_VERSION}.${DISTRO_MINOR_VERSION}" - elif [ "${DISTRO_MAJOR_VERSION}" -lt 42 ]; then - case ${DISTRO_MAJOR_VERSION} in - 15) DISTRO_REPO="openSUSE_Leap_${DISTRO_MAJOR_VERSION}.${DISTRO_MINOR_VERSION}" - ;; - *) DISTRO_REPO="SLE_${DISTRO_MAJOR_VERSION}_SP${SUSE_PATCHLEVEL}" - ;; - esac + else + DISTRO_REPO="SLE_${DISTRO_MAJOR_VERSION}_SP${SUSE_PATCHLEVEL}" fi if [ "$_DOWNSTREAM_PKG_REPO" -eq $BS_TRUE ]; then @@ -5688,8 +5683,7 @@ install_opensuse_git_post() { if [ -f /bin/systemctl ]; then use_usr_lib=$BS_FALSE - if [ "${DISTRO_MAJOR_VERSION}" -gt 13 ] || \ - { [ "${DISTRO_MAJOR_VERSION}" -eq 13 ] && [ "${DISTRO_MINOR_VERSION}" -ge 2 ]; }; then + if [ "${DISTRO_MAJOR_VERSION}" -ge 15 ]; then use_usr_lib=$BS_TRUE fi From a1201d1d282ebf88eff192d4ede4660f98f1780b Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 12 Jul 2018 17:39:00 -0400 Subject: [PATCH 45/65] Add opensuse 15 specific installation functions --- README.rst | 9 +++ bootstrap-salt.sh | 142 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 121 insertions(+), 30 deletions(-) diff --git a/README.rst b/README.rst index 638b3c8..c5b4573 100644 --- a/README.rst +++ b/README.rst @@ -266,10 +266,19 @@ Red Hat family SUSE family ~~~~~~~~~~~ +- openSUSE Leap 15 (see note below) - openSUSE Leap 42.3 - openSUSE Tumbleweed 2015 - SUSE Linux Enterprise Server 11 SP4, 12 SP2 +**NOTE:** Leap 15 installs Python 3 Salt packages by default. Salt is packaged by SUSE, and +Leap 15 ships with Python 3. Salt with Python 2 can be installed using the the ``-x`` option +in combination with the ``git`` installation method. + +.. code:: console + + sh bootstrap-salt.sh -x python2 git v2018.3.2 + Ubuntu and derivatives ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index c50d89e..7e9f6a3 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -5537,7 +5537,8 @@ __zypper_install() { fi } -install_opensuse_stable_deps() { +__opensuse_prep_install() { + # DRY function for common installation preparatory steps for SUSE if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then # Is the repository already known __set_suse_pkg_repo @@ -5546,6 +5547,7 @@ install_opensuse_stable_deps() { fi __zypper --gpg-auto-import-keys refresh + # shellcheck disable=SC2181 if [ $? -ne 0 ] && [ $? -ne 4 ]; then # If the exit code is not 0, and it's not 4 (failed to update a @@ -5553,15 +5555,19 @@ install_opensuse_stable_deps() { return 1 fi + if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then + __zypper --gpg-auto-import-keys update || return 1 + fi +} + +install_opensuse_stable_deps() { + __opensuse_prep_install || return 1 + if [ "$DISTRO_MAJOR_VERSION" -eq 12 ] && [ "$DISTRO_MINOR_VERSION" -eq 3 ]; then # Because patterns-openSUSE-minimal_base-conflicts conflicts with python, lets remove the first one __zypper remove patterns-openSUSE-minimal_base-conflicts fi - if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then - __zypper --gpg-auto-import-keys update || return 1 - fi - # YAML module is used for generating custom master/minion configs # requests is still used by many salt modules # Salt needs python-zypp installed in order to use the zypper module @@ -5757,24 +5763,112 @@ install_opensuse_check_services() { # ####################################################################################################################### +####################################################################################################################### +# +# openSUSE Leap 15 +# + +install_opensuse_15_stable_deps() { + __opensuse_prep_install || return 1 + + # SUSE only packages Salt for Python 3 on Leap 15 + # Py3 is the default bootstrap install for Leap 15 + # However, git installs might specify "-x python2" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 2 ]; then + PY_PKG_VER=2 + else + PY_PKG_VER=3 + fi + + # YAML module is used for generating custom master/minion configs + # requests is still used by many salt modules + __PACKAGES="python${PY_PKG_VER}-PyYAML python${PY_PKG_VER}-requests" + + # shellcheck disable=SC2086 + __zypper_install ${__PACKAGES} || return 1 + + if [ "${_EXTRA_PACKAGES}" != "" ]; then + echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" + # shellcheck disable=SC2086 + __zypper_install ${_EXTRA_PACKAGES} || return 1 + fi + + return 0 +} + +install_opensuse_15_git_deps() { + install_opensuse_15_stable_deps || return 1 + + if ! __check_command_exists git; then + __zypper_install git || return 1 + fi + + __git_clone_and_checkout || return 1 + + # Py3 is the default bootstrap install for Leap 15 + # However, git installs might specify "-x python2" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 2 ]; then + PY_PKG_VER=2 + + # This is required by some of the python2 packages below + __PACKAGES="libpython2_7-1_0" + else + PY_PKG_VER=3 + __PACKAGES="" + fi + + __PACKAGES="${__PACKAGES} libzmq5 python${PY_PKG_VER}-Jinja2 python${PY_PKG_VER}-msgpack" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-pycrypto python${PY_PKG_VER}-pyzmq" + + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" ]; then + # We're on the develop branch, install whichever tornado is on the requirements file + __REQUIRED_TORNADO="$(grep tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt")" + if [ "${__REQUIRED_TORNADO}" != "" ]; then + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-tornado" + fi + fi + + if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-apache-libcloud" + fi + + # shellcheck disable=SC2086 + __zypper_install ${__PACKAGES} || return 1 + + # Let's trigger config_salt() + if [ "$_TEMP_CONFIG_DIR" = "null" ]; then + _TEMP_CONFIG_DIR="${_SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_opensuse_15_git() { + + # Py3 is the default bootstrap install for Leap 15 + if [ -n "$_PY_EXE" ]; then + _PYEXE=${_PY_EXE} + else + _PYEXE=python3 + fi + + ${_PYEXE} setup.py ${SETUP_PY_INSTALL_ARGS} install --prefix=/usr || return 1 + return 0 +} + +# +# End of openSUSE Leap 15 +# +####################################################################################################################### + ####################################################################################################################### # # SUSE Enterprise 12 # install_suse_12_stable_deps() { - if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then - # Is the repository already known - __set_suse_pkg_repo - # Check zypper repos and refresh if necessary - __check_and_refresh_suse_pkg_repo - fi - - __zypper --gpg-auto-import-keys refresh || return 1 - - if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then - __zypper --gpg-auto-import-keys update || return 1 - fi + __opensuse_prep_install || return 1 # YAML module is used for generating custom master/minion configs # requests is still used by many salt modules @@ -5793,7 +5887,6 @@ install_suse_12_stable_deps() { # Let's try to install the higher version first and then the lower one in case of failure __zypper_install 'python-M2Crypto>=0.22' || __zypper_install 'python-M2Crypto>=0.21' || return 1 - if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" # shellcheck disable=SC2086 @@ -5877,18 +5970,7 @@ install_suse_12_restart_daemons() { # install_suse_11_stable_deps() { - if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then - # Is the repository already known - __set_suse_pkg_repo - # Check zypper repos and refresh if necessary - __check_and_refresh_suse_pkg_repo - fi - - __zypper --gpg-auto-import-keys refresh || return 1 - - if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then - __zypper --gpg-auto-import-keys update || return 1 - fi + __opensuse_prep_install || return 1 # YAML module is used for generating custom master/minion configs __PACKAGES="python-PyYAML" From 139583befdbfe9d3c6924eb9e8f2c886399707dd Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 13 Jul 2018 14:26:10 -0400 Subject: [PATCH 46/65] Make some initial preparations for Ubuntu 18.10 support Ubuntu 18.10 is not released yet, but we can make some small changes now to make the script a little more robust for when it is released. --- README.rst | 4 ++-- bootstrap-salt.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 60c7bed..3c606b4 100644 --- a/README.rst +++ b/README.rst @@ -286,8 +286,8 @@ repositories are not provided on `SaltStack's Ubuntu repository`_ for the non-LT bootstrap script will attempt to install the packages for the most closely related LTS Ubuntu release instead. -For example, when installing Salt on Ubuntu 17.10, the bootstrap script will setup the repository -for Ubuntu 16.04 from `SaltStack's Ubuntu repository`_ and install the 16.04 packages. +For example, when installing Salt on Ubuntu 18.10, the bootstrap script will setup the repository +for Ubuntu 18.04 from `SaltStack's Ubuntu repository`_ and install the 18.04 packages. Non-LTS Ubuntu releases are not supported once the release reaches End-of-Life as defined by `Ubuntu's release schedule`_. diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 1e70776..af82ef6 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2573,7 +2573,7 @@ __enable_universe_repository() { __install_saltstack_ubuntu_repository() { # Workaround for latest non-LTS ubuntu - if [ "$DISTRO_MAJOR_VERSION" -gt 18 ]; then + if [ "$DISTRO_MAJOR_VERSION" -gt 18 ] || [ "${DISTRO_VERSION}" -eq "18.10" ]; then echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages for latest LTS release. You may experience problems." UBUNTU_VERSION=18.04 UBUNTU_CODENAME="bionic" From 4ae8b53513f1d944e2002d317da64f53c0ce573c Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 20 Jul 2018 13:01:35 -0400 Subject: [PATCH 47/65] Update changelog --- ChangeLog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ChangeLog b/ChangeLog index faa0755..d1a5fad 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,12 @@ Version TBD (In Progress on the Develop Branch): + * Add opensuse 15 specific installation functions (rallytime) #1263 + * Remove support for openSUSE Leap 42.2 (rallytime) #1262 + * Remove support for Fedora 26 (rallytime) #1261 + * Remove support for Ubuntu 17.10 (rallytime) #1260 + * Ensure pipe is absent before mkfifo/mknod (noelmcloughlin) #1256 + * fix for opensuse15 if no lsb_release pkg (noelmcloughlin) #1255 + * Fix Up ShellCheck errors/warnings for latest version of shellcheck (rallytime) #1253 + * Remove daily install option on Ubuntu system (icy) #1250 * Add python 3 support for Ubuntu 18 (rallytime) #1248 * Return immediately if apt-get fails (icy) #1247 * Disable shell error about unbound variable during daily install (icy) #1246 From bc6c3e5b3b69912700bd180758dfd36917008223 Mon Sep 17 00:00:00 2001 From: Brett Benassi Date: Fri, 20 Jul 2018 15:32:22 -0600 Subject: [PATCH 48/65] Updting checkstyle to fail on warnings --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 84d6ae8..6242301 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -72,7 +72,7 @@ node ('bootstrap') { stage('checkout') { checkout scm } stage('shellcheck') { sh 'stack exec -- shellcheck -s sh -f checkstyle bootstrap-salt.sh | tee checkstyle.xml' - checkstyle pattern: '**/checkstyle.xml' + checkstyle canComputeNew: false, defaultEncoding: '', healthy: '', pattern: '**/checkstyle.xml', unHealthy: '', unstableTotalAll: '0' archiveArtifacts artifacts: '**/checkstyle.xml' } // if (env.CHANGE_ID) { From b899014f1cee412ff3d4ac3d2ab1c6724fc03c83 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 23 Jul 2018 10:38:02 -0400 Subject: [PATCH 49/65] Fix lint error: don't use decimals --- bootstrap-salt.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 889867e..7a043b7 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2577,7 +2577,8 @@ __enable_universe_repository() { __install_saltstack_ubuntu_repository() { # Workaround for latest non-LTS ubuntu - if [ "$DISTRO_MAJOR_VERSION" -gt 18 ] || [ "${DISTRO_VERSION}" -eq "18.10" ]; then + if [ "$DISTRO_MAJOR_VERSION" -gt 18 ] || \ + { [ "$DISTRO_MAJOR_VERSION" -eq 18 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; }; then echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages for latest LTS release. You may experience problems." UBUNTU_VERSION=18.04 UBUNTU_CODENAME="bionic" From 50c2dd3c3774d54ec0fc0e897d21532018a6ed23 Mon Sep 17 00:00:00 2001 From: Michiel Pouw Date: Wed, 25 Jul 2018 13:00:53 +0200 Subject: [PATCH 50/65] SLES12SP changed packages git to git-core and libzmq3 to libzmq4 --- bootstrap-salt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 7a043b7..4ffb0ec 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -5894,14 +5894,14 @@ install_suse_12_git_deps() { install_suse_12_stable_deps || return 1 if ! __check_command_exists git; then - __zypper_install git || return 1 + __zypper_install git-core || return 1 fi __git_clone_and_checkout || return 1 __PACKAGES="" # shellcheck disable=SC2089 - __PACKAGES="${__PACKAGES} libzmq3 python-Jinja2 python-msgpack-python python-pycrypto" + __PACKAGES="${__PACKAGES} libzmq4 python-Jinja2 python-msgpack-python python-pycrypto" __PACKAGES="${__PACKAGES} python-pyzmq python-xml" if [ -f "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" ]; then From 6ad614bd4a5e5042546508871d32ed7f30d8ff73 Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Sun, 24 Jun 2018 19:41:20 +0200 Subject: [PATCH 51/65] Allow arm64 installation on Debian systems Previously, AArch64 machines ('arm64' DPKG architecture) were not supported due to missing repository metadata. - relax bail-out condition in DPKG machine architecture check to allow arm64; print a warning about experimental support; - if no custom repo is defined, default to re-using amd64 repo on arm64 since all packages are arch-independent; Fixes: https://github.com/saltstack/salt-bootstrap/issues/1240 Signed-off-by: Alexandru Avadanii --- bootstrap-salt.sh | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 7a043b7..b489267 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -1315,6 +1315,7 @@ __check_dpkg_architecture() { fi __REPO_ARCH="$DPKG_ARCHITECTURE" + __REPO_ARCH_DEB='deb' __return_code=0 case $DPKG_ARCHITECTURE in @@ -1326,6 +1327,18 @@ __check_dpkg_architecture() { "amd64") error_msg="" ;; + "arm64") + if [ "$_CUSTOM_REPO_URL" != "null" ]; then + warn_msg="Support for arm64 is experimental, make sure the custom repository used has the expected structure and contents." + else + # Saltstack official repository does not yet have arm64 metadata, + # use amd64 repositories on arm64, since all pkgs are arch-independent + __REPO_ARCH="amd64" + __REPO_ARCH_DEB="deb [arch=$__REPO_ARCH]" + warn_msg="Support for arm64 packages is experimental and might rely on architecture-independent packages from the amd64 repository." + fi + error_msg="" + ;; "armhf") if [ "$DISTRO_NAME_L" = "ubuntu" ] || [ "$DISTRO_MAJOR_VERSION" -lt 8 ]; then error_msg="Support for armhf packages at $_REPO_URL is limited to Debian/Raspbian 8 platforms." @@ -1340,6 +1353,11 @@ __check_dpkg_architecture() { ;; esac + if [ "${warn_msg}" != "" ]; then + # AArch64: Do not fail at this point, but warn the user about experimental support + # See https://github.com/saltstack/salt-bootstrap/issues/1240 + echowarn "${warn_msg}" + fi if [ "${error_msg}" != "" ]; then echoerror "${error_msg}" if [ "$ITYPE" != "git" ]; then @@ -2611,7 +2629,7 @@ __install_saltstack_ubuntu_repository() { # SaltStack's stable Ubuntu repository: SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${__PY_VERSION_REPO}/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/${STABLE_REV}" - echo "deb $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/saltstack.list + echo "$__REPO_ARCH_DEB $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/saltstack.list __apt_key_fetch "$SALTSTACK_UBUNTU_URL/SALTSTACK-GPG-KEY.pub" || return 1 @@ -3010,7 +3028,7 @@ __install_saltstack_debian_repository() { # amd64 is just a part of repository URI, 32-bit pkgs are hosted under the same location SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/${STABLE_REV}" - echo "deb $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/saltstack.list" + echo "$__REPO_ARCH_DEB $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/saltstack.list" __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALTSTACK-GPG-KEY.pub" || return 1 From c82bf41f5e5e65cda264580ecaf013650f9a7dcd Mon Sep 17 00:00:00 2001 From: William Giokas <1007380@gmail.com> Date: Mon, 30 Jul 2018 14:59:20 -0600 Subject: [PATCH 52/65] Adds timestamps and colors to jenkins --- Jenkinsfile | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6242301..b135966 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -69,23 +69,27 @@ def prSetupRuns = prDistroversions.collectEntries { } node ('bootstrap') { - stage('checkout') { checkout scm } - stage('shellcheck') { - sh 'stack exec -- shellcheck -s sh -f checkstyle bootstrap-salt.sh | tee checkstyle.xml' - checkstyle canComputeNew: false, defaultEncoding: '', healthy: '', pattern: '**/checkstyle.xml', unHealthy: '', unstableTotalAll: '0' - archiveArtifacts artifacts: '**/checkstyle.xml' + timestamps { + ansiColor('xterm') { + stage('checkout') { checkout scm } + stage('shellcheck') { + sh 'stack exec -- shellcheck -s sh -f checkstyle bootstrap-salt.sh | tee checkstyle.xml' + checkstyle canComputeNew: false, defaultEncoding: '', healthy: '', pattern: '**/checkstyle.xml', unHealthy: '', unstableTotalAll: '0' + archiveArtifacts artifacts: '**/checkstyle.xml' + } + // if (env.CHANGE_ID) { + // // Running for a PR only runs against 4 random distros from a shorter list + // stage('kitchen-pr') { + // parallel prSetupRuns + // } + // } else { + // // If we're not running for a pr we run *everything* + // stage('kitchen-all') { + // parallel setupRuns + // } + // } + } } - // if (env.CHANGE_ID) { - // // Running for a PR only runs against 4 random distros from a shorter list - // stage('kitchen-pr') { - // parallel prSetupRuns - // } - // } else { - // // If we're not running for a pr we run *everything* - // stage('kitchen-all') { - // parallel setupRuns - // } - // } } /* From 3b7d2fee2bb31eb93e34aba0fab2bba283ae7025 Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Sat, 4 Aug 2018 03:54:00 +0200 Subject: [PATCH 53/65] Fix undefined variable warn_msg on amd64 Fixes: 6ad614bd Signed-off-by: Alexandru Avadanii --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 7279aa0..6005a83 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -1353,7 +1353,7 @@ __check_dpkg_architecture() { ;; esac - if [ "${warn_msg}" != "" ]; then + if [ "${warn_msg:-}" != "" ]; then # AArch64: Do not fail at this point, but warn the user about experimental support # See https://github.com/saltstack/salt-bootstrap/issues/1240 echowarn "${warn_msg}" From 75c6ca4af676660c901418d317deceedf13da314 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 7 Aug 2018 13:12:45 -0400 Subject: [PATCH 54/65] [Arch] Add python2-futures to list of pkgs on git install python2-futures is listed as an optional dependency, but is required by python2-tornado. We need to install it along with everything else. --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 6005a83..723cbad 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4723,7 +4723,7 @@ install_arch_linux_git_deps() { fi pacman -R --noconfirm python2-distribute pacman -Su --noconfirm --needed python2-crypto python2-setuptools python2-jinja \ - python2-m2crypto python2-markupsafe python2-msgpack python2-psutil \ + python2-m2crypto python2-futures cpython2-markupsafe python2-msgpack python2-psutil \ python2-pyzmq zeromq python2-requests python2-systemd || return 1 __git_clone_and_checkout || return 1 From afd39f4e1d839cf38716c67d1c9619518d3b13dd Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 7 Aug 2018 13:26:53 -0400 Subject: [PATCH 55/65] Remove typo: extra 'c' was accidentally added in #1269 --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 723cbad..70f844b 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4723,7 +4723,7 @@ install_arch_linux_git_deps() { fi pacman -R --noconfirm python2-distribute pacman -Su --noconfirm --needed python2-crypto python2-setuptools python2-jinja \ - python2-m2crypto python2-futures cpython2-markupsafe python2-msgpack python2-psutil \ + python2-m2crypto python2-futures python2-markupsafe python2-msgpack python2-psutil \ python2-pyzmq zeromq python2-requests python2-systemd || return 1 __git_clone_and_checkout || return 1 From da7ee8433d6366e45aa22dc39d53aca2f1accf58 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 7 Aug 2018 15:56:54 -0400 Subject: [PATCH 56/65] Handle commented lines in the requirements files for pip pkgs Fixes an issue in CentOS 6 where pip pkgs weren't getting installed because of the comments added to the requirements files in Salt. --- bootstrap-salt.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 70f844b..401a6bc 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3788,8 +3788,10 @@ install_centos_git_deps() { fi if [ -f "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" ]; then - for SINGLE_PACKAGE in $_PIP_PACKAGES; do - __REQUIRED_VERSION="$(grep "${SINGLE_PACKAGE}" "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt")" + # Filter out any commented lines from the requirements file + _REQ_LINES="$(grep '^[^#]' "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt")" + for SINGLE_PACKAGE in ${_PIP_PACKAGES}; do + __REQUIRED_VERSION="$(grep "${SINGLE_PACKAGE}" "${_REQ_LINES}")" if [ "${__REQUIRED_VERSION}" != "" ]; then _PIP_PACKAGES=$(echo "$_PIP_PACKAGES" | sed "s/${SINGLE_PACKAGE}/${__REQUIRED_VERSION}/") fi From bfb7c794651b1512d17334f1ab87059e3cf3dac2 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Thu, 9 Aug 2018 12:39:30 +0200 Subject: [PATCH 57/65] Fedora Py3 compat --- bootstrap-salt.sh | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 401a6bc..72ac705 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3415,9 +3415,22 @@ install_debian_check_services() { # install_fedora_deps() { + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + # Packages are named python3- + PY_PKG_VER=3 + __PACKAGES="python3-m2crypto python3-PyYAML" + else + PY_PKG_VER=2 + __PACKAGES="m2crypto" + if [ "$DISTRO_MAJOR_VERSION" -ge 28 ]; then + __PACKAGES="${__PACKAGES} python2-pyyaml" + else + __PACKAGES="${__PACKAGES} PyYAML" + fi + fi - __PACKAGES="dnf-utils libyaml m2crypto PyYAML python-crypto python-jinja2" - __PACKAGES="${__PACKAGES} python2-msgpack python2-requests python-zmq" + __PACKAGES="dnf-utils ${__PACKAGES} python${PY_PKG_VER}-crypto python${PY_PKG_VER}-requests python${PY_PKG_VER}-zmq" + __PACKAGES="${__PACKAGES} libyaml python${PY_PKG_VER}-jinja2 python${PY_PKG_VER}-msgpack" # shellcheck disable=SC2086 dnf install -y ${__PACKAGES} || return 1 @@ -3474,6 +3487,12 @@ install_fedora_stable_post() { } install_fedora_git_deps() { + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + # Packages are named python3- + PY_PKG_VER=3 + else + PY_PKG_VER=2 + fi if [ "$_INSECURE_DL" -eq $BS_FALSE ] && [ "${_SALT_REPO_URL%%://*}" = "https" ]; then dnf install -y ca-certificates || return 1 @@ -3487,10 +3506,10 @@ install_fedora_git_deps() { __git_clone_and_checkout || return 1 - __PACKAGES="python2-tornado systemd-python" + __PACKAGES="python${PY_PKG_VER}-tornado python${PY_PKG_VER}-systemd" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} python-libcloud python-netaddr" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-libcloud python${PY_PKG_VER}-netaddr" fi # shellcheck disable=SC2086 @@ -3506,10 +3525,17 @@ install_fedora_git_deps() { } install_fedora_git() { - if [ -f "${_SALT_GIT_CHECKOUT_DIR}/salt/syspaths.py" ]; then - python setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR}" ${SETUP_PY_INSTALL_ARGS} install || return 1 + if [ "${_PY_EXE}" != "" ]; then + _PYEXE=${_PY_EXE} + echoinfo "Using the following python version: ${_PY_EXE} to install salt" else - python setup.py ${SETUP_PY_INSTALL_ARGS} install || return 1 + _PYEXE='python2' + fi + + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/salt/syspaths.py" ]; then + ${_PYEXE} setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR}" ${SETUP_PY_INSTALL_ARGS} install --prefix=/usr || return 1 + else + ${_PYEXE} setup.py ${SETUP_PY_INSTALL_ARGS} install --prefix=/usr || return 1 fi return 0 } From 65d134132c37723700683c042a8eae55eb383f5f Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Thu, 9 Aug 2018 14:38:07 +0200 Subject: [PATCH 58/65] Fedora Py3 Tornado <5.0 workaround --- bootstrap-salt.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 72ac705..407f977 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3506,7 +3506,15 @@ install_fedora_git_deps() { __git_clone_and_checkout || return 1 - __PACKAGES="python${PY_PKG_VER}-tornado python${PY_PKG_VER}-systemd" + __PACKAGES="python${PY_PKG_VER}-systemd" + if [ "${PY_PKG_VER}" -eq 3 -a "$DISTRO_MAJOR_VERSION" -ge 28 ]; then + __check_pip_allowed "You need to allow pip based installations (-P) for Tornado <5.0 in order to install Salt on Python 3" + grep tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" | while IFS=$'\n' read -r dep; do + "${_PY_EXE}" -m pip install "${dep}" || return 1 + done + else + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-tornado" + fi if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-libcloud python${PY_PKG_VER}-netaddr" From 4dbf02c1a560afce1eb7ed8c9a1a2c5f63858aba Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Thu, 9 Aug 2018 18:50:09 +0200 Subject: [PATCH 59/65] Fedora Py3 docs --- README.rst | 5 +++++ bootstrap-salt.sh | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 8c9ac3b..4007966 100644 --- a/README.rst +++ b/README.rst @@ -342,11 +342,16 @@ Python 3 Support Some distributions support installing Salt to use Python 3 instead of Python 2. The availability of this offering, while limited, is as follows: +- CentOS 6 - CentOS 7 - Debian 9 +- Fedora - Ubuntu 16.04 - Ubuntu 18.04 +The Fedora and CentOS 6 options only support git installations. +On Fedora 28, PIP installation must be allowed (-P) due to incompatibility with the shipped Tornado library. + Installing the Python 3 packages for Salt is done via the ``-x`` option: .. code:: console diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 407f977..549b703 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -381,8 +381,8 @@ __usage() { a complete overwrite of the file. -q Quiet salt installation from git (setup.py install -q) -x Changes the Python version used to install Salt. Currently, this is only - supported on CentOS 7, Debian 9, Ubuntu 16 and CentOS 6. The CentOS 6 - option only works with git installations. + supported on CentOS 6+, Debian 9+, Ubuntu 16+, and Fedora. + The Fedora and CentOS 6 options only work with git installations. -y Installs a different python version on host. Currently this has only been tested with CentOS 6 and is considered experimental. This will install the ius repo on the box if disable repo is false. This must be used in conjunction From cbf429653131b7809eea67420e2ffac78bd1bc48 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Thu, 9 Aug 2018 18:55:43 +0200 Subject: [PATCH 60/65] Fedora Py3 shellcheck fixes --- bootstrap-salt.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 549b703..9df7669 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3507,8 +3507,9 @@ install_fedora_git_deps() { __git_clone_and_checkout || return 1 __PACKAGES="python${PY_PKG_VER}-systemd" - if [ "${PY_PKG_VER}" -eq 3 -a "$DISTRO_MAJOR_VERSION" -ge 28 ]; then + if [ "${PY_PKG_VER}" -eq 3 ] && [ "$DISTRO_MAJOR_VERSION" -ge 28 ]; then __check_pip_allowed "You need to allow pip based installations (-P) for Tornado <5.0 in order to install Salt on Python 3" + # shellcheck disable=SC2039 grep tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" | while IFS=$'\n' read -r dep; do "${_PY_EXE}" -m pip install "${dep}" || return 1 done From b24e88430777356e3902ceaf2892d51f1db29d6e Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Thu, 9 Aug 2018 19:12:42 +0200 Subject: [PATCH 61/65] Fedora Py3 POSIX fix --- bootstrap-salt.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 9df7669..bc1187b 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3509,9 +3509,9 @@ install_fedora_git_deps() { __PACKAGES="python${PY_PKG_VER}-systemd" if [ "${PY_PKG_VER}" -eq 3 ] && [ "$DISTRO_MAJOR_VERSION" -ge 28 ]; then __check_pip_allowed "You need to allow pip based installations (-P) for Tornado <5.0 in order to install Salt on Python 3" - # shellcheck disable=SC2039 - grep tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" | while IFS=$'\n' read -r dep; do - "${_PY_EXE}" -m pip install "${dep}" || return 1 + grep tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" | while IFS=' +' read -r dep; do + "${_PY_EXE}" -m pip install "${dep}" || return 1 done else __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-tornado" From 3785e67c43c1a03c1c08392afaf97209e9f0b910 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Thu, 9 Aug 2018 21:03:18 +0200 Subject: [PATCH 62/65] fedora deps sort alphabetically --- bootstrap-salt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index bc1187b..90e6c2a 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3429,8 +3429,8 @@ install_fedora_deps() { fi fi - __PACKAGES="dnf-utils ${__PACKAGES} python${PY_PKG_VER}-crypto python${PY_PKG_VER}-requests python${PY_PKG_VER}-zmq" - __PACKAGES="${__PACKAGES} libyaml python${PY_PKG_VER}-jinja2 python${PY_PKG_VER}-msgpack" + __PACKAGES="${__PACKAGES} dnf-utils libyaml python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests python${PY_PKG_VER}-zmq" # shellcheck disable=SC2086 dnf install -y ${__PACKAGES} || return 1 From b8e6a96451e79e4e61da32a33f7870ee6eb1cd42 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Thu, 9 Aug 2018 21:12:57 +0200 Subject: [PATCH 63/65] Py3/-x doc clarification --- README.rst | 4 +--- bootstrap-salt.sh | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 4007966..ebe66ef 100644 --- a/README.rst +++ b/README.rst @@ -342,14 +342,12 @@ Python 3 Support Some distributions support installing Salt to use Python 3 instead of Python 2. The availability of this offering, while limited, is as follows: -- CentOS 6 - CentOS 7 - Debian 9 -- Fedora +- Fedora (only git installations) - Ubuntu 16.04 - Ubuntu 18.04 -The Fedora and CentOS 6 options only support git installations. On Fedora 28, PIP installation must be allowed (-P) due to incompatibility with the shipped Tornado library. Installing the Python 3 packages for Salt is done via the ``-x`` option: diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 90e6c2a..9323932 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -380,9 +380,9 @@ __usage() { no ".bak" file will be created as either of those options will force a complete overwrite of the file. -q Quiet salt installation from git (setup.py install -q) - -x Changes the Python version used to install Salt. Currently, this is only - supported on CentOS 6+, Debian 9+, Ubuntu 16+, and Fedora. - The Fedora and CentOS 6 options only work with git installations. + -x Changes the Python version used to install Salt. + For CentOS 6 git installations python2.7 is supported. + Fedora git installation, CentOS 7, Debian 9, Ubuntu 16.04 and 18.04 support python3. -y Installs a different python version on host. Currently this has only been tested with CentOS 6 and is considered experimental. This will install the ius repo on the box if disable repo is false. This must be used in conjunction From 633334e98a0b8306c8189f25d15cdc96ac25faf3 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 13 Aug 2018 13:39:12 -0400 Subject: [PATCH 64/65] Update AUTHORS and ChangeLog for release --- AUTHORS.rst | 2 ++ ChangeLog | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index 8159084..cd7c28f 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -12,6 +12,7 @@ Adam Wright hipikat adam@hipikat.org Alec Koumjian akoumjian akoumjian@gmail.com Alex Van't Hof alexvh Alexander Krasnukhin themalkolm the.malkolm@gmail.com +Alexandru Avadanii alexandruavadanii Alexandru.Avadanii@enea.com Alexey dmitrievav Ali Rizvi-Santiago arizvisa amendlik amendlik @@ -98,6 +99,7 @@ Matthew Mead-Briggs mattmb Matthew Richardson mrichar1 Matthew Willson ixela Matthieu Guegan mguegan +mfapouw mfapouw Michael A. Smith kojiromike michaels@syapse.com Michael Scherer mscherer Michele Bologna mbologna michele.bologna@gmail.com diff --git a/ChangeLog b/ChangeLog index d1a5fad..76195fa 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,12 @@ Version TBD (In Progress on the Develop Branch): + +Version 2018.08.13: + * Fedora Py3 fixes (The-Loeki) #1273 + * Handle commented lines in the requirements files for pip pkgs (rallytime) #1271 + * Remove typo: extra 'c' was accidentally added in #1269 (rallytime) #1270 + * [Arch] Add python2-futures to list of pkgs on git install (rallytime) #1269 + * Fix undefined variable warn_msg on amd64 (alexandruavadanii) #1268 + * SLES12SP changed packages git to git-core and libzmq3 to libzmq4 (mfapouw) #1266 * Add opensuse 15 specific installation functions (rallytime) #1263 * Remove support for openSUSE Leap 42.2 (rallytime) #1262 * Remove support for Fedora 26 (rallytime) #1261 From 64d1a3573dc55bd376891fc56a2081b5d88f40ae Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 13 Aug 2018 13:43:22 -0400 Subject: [PATCH 65/65] Update bootstrap version number --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 401a6bc..d31acf5 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -18,7 +18,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2018.04.25" +__ScriptVersion="2018.08.13" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0"