From 35cd8046a2cc79ffa38f1b5191c4421be3cf0e3f Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Sat, 23 Apr 2016 14:57:45 +0300 Subject: [PATCH 1/8] Fix bootstrapping from git on Debian 8 by installing latest `tornado` via pip --- bootstrap-salt.sh | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 83be1a1..e16ad69 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2796,6 +2796,7 @@ install_debian_8_deps() { if [ $_START_DAEMONS -eq $BS_FALSE ]; then echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." fi + # No user interaction, libc6 restart services for example export DEBIAN_FRONTEND=noninteractive @@ -2892,7 +2893,7 @@ install_debian_git_deps() { fi if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then - __check_pip_allowed "You need to allow pip based installations (-P) in order to install apache-libcloud" + __check_pip_allowed "You need to allow pip based installations (-P) in order to install requested 'apache-libcloud" pip install -U "apache-libcloud>=$_LIBCLOUD_MIN_VERSION" fi @@ -2962,15 +2963,23 @@ install_debian_8_git_deps() { fi __apt_get_install_noinput lsb-release python python-pkg-resources python-crypto \ - python-jinja2 python-m2crypto python-yaml msgpack-python python-pip || return 1 + python-jinja2 python-m2crypto python-yaml msgpack-python || return 1 __git_clone_and_checkout || return 1 if [ -f "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" ]; then - # We're on the develop branch, install tornado __REQUIRED_TORNADO="$(grep tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt")" - if [ "${__REQUIRED_TORNADO}" != "" ]; then - __apt_get_install_noinput python-tornado + if [ -n "${__REQUIRED_TORNADO}" ]; then + __check_pip_allowed "You need to allow pip based installations (-P) in order to install required 'tornado' package" + + __PACKAGES="python-dev" + if ! __check_command_exists pip; then + __PACKAGES="${__PACKAGES} python-pip" + fi + + # shellcheck disable=SC2086 + __apt_get_install_noinput ${__PACKAGES} || return 1 + pip install -U tornado || return 1 fi fi @@ -3054,12 +3063,17 @@ install_debian_git_post() { # Skip if not meant to be installed [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue - [ $fname = "api" ] && ([ "$_INSTALL_MASTER" -eq $BS_FALSE ] || ! __check_command_exists "salt-${fname}") && continue + [ $fname = "api" ] && \ + ([ "$_INSTALL_MASTER" -eq $BS_FALSE ] || ! __check_command_exists "salt-${fname}") && \ + continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue if [ -f /bin/systemctl ]; then - if [ ! -f /etc/systemd/system/salt-${fname}.service ] || ([ -f /etc/systemd/system/salt-${fname}.service ] && [ $_FORCE_OVERWRITE -eq $BS_TRUE ]); then - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" /etc/systemd/system + if [ ! -f /lib/systemd/system/salt-${fname}.service ] || \ + ([ -f /lib/systemd/system/salt-${fname}.service ] && [ $_FORCE_OVERWRITE -eq $BS_TRUE ]); then + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" /lib/systemd/system + # FIXME: add Debian-specific unit files to the salt main repo + sed -i -e '/^Type/ s/notify/simple/' /lib/systemd/system/salt-${fname}.service fi # Skip salt-api since the service should be opt-in and not necessarily started on boot From 0bcda90daca43ca2a03eccf0fd263a7210de97c3 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Mon, 25 Apr 2016 11:22:49 +0300 Subject: [PATCH 2/8] Close singe quote in the message. Thanks, @s0undt3ch --- bootstrap-salt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index e16ad69..8fad023 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2893,7 +2893,7 @@ install_debian_git_deps() { fi if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then - __check_pip_allowed "You need to allow pip based installations (-P) in order to install requested 'apache-libcloud" + __check_pip_allowed "You need to allow pip based installations (-P) in order to install requested 'apache-libcloud' module" pip install -U "apache-libcloud>=$_LIBCLOUD_MIN_VERSION" fi @@ -2970,7 +2970,7 @@ install_debian_8_git_deps() { if [ -f "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" ]; then __REQUIRED_TORNADO="$(grep tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt")" if [ -n "${__REQUIRED_TORNADO}" ]; then - __check_pip_allowed "You need to allow pip based installations (-P) in order to install required 'tornado' package" + __check_pip_allowed "You need to allow pip based installations (-P) in order to install required 'tornado' module" __PACKAGES="python-dev" if ! __check_command_exists pip; then From ce3b96b14a7b0b2aba3db7bc1019eeda469706e5 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Wed, 27 Apr 2016 11:36:04 +0300 Subject: [PATCH 3/8] Use initscripts and units from Git working directory if present --- bootstrap-salt.sh | 48 +++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 8fad023..da3652c 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2951,6 +2951,7 @@ install_debian_7_git_deps() { install_debian_8_git_deps() { install_debian_8_deps || return 1 + # No user interaction, libc6 restart services for example export DEBIAN_FRONTEND=noninteractive @@ -3061,43 +3062,54 @@ install_debian_git_post() { for fname in minion master syndic api; do # Skip if not meant to be installed - [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue - [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue - [ $fname = "api" ] && \ + [ "$fname" = "api" ] && \ ([ "$_INSTALL_MASTER" -eq $BS_FALSE ] || ! __check_command_exists "salt-${fname}") && \ continue - [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + [ "$fname" = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue + [ "$fname" = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue + [ "$fname" = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + # Configure SystemD for Debian 8 "Jessie" and later if [ -f /bin/systemctl ]; then if [ ! -f /lib/systemd/system/salt-${fname}.service ] || \ ([ -f /lib/systemd/system/salt-${fname}.service ] && [ $_FORCE_OVERWRITE -eq $BS_TRUE ]); then - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" /lib/systemd/system - # FIXME: add Debian-specific unit files to the salt main repo - sed -i -e '/^Type/ s/notify/simple/' /lib/systemd/system/salt-${fname}.service + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.service" ]; then + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.service" /lib/systemd/system + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.environment" "/etc/default/salt-${fname}" + else + # workaround before adding Debian-specific unit files to the Salt main repo + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" /lib/systemd/system + sed -i -e '/^Type/ s/notify/simple/' /lib/systemd/system/salt-${fname}.service + fi fi # Skip salt-api since the service should be opt-in and not necessarily started on boot - [ $fname = "api" ] && continue + [ "$fname" = "api" ] && continue - /bin/systemctl enable salt-${fname}.service + /bin/systemctl enable "salt-${fname}.service" SYSTEMD_RELOAD=$BS_TRUE - elif [ ! -f /etc/init.d/salt-$fname ] || ([ -f /etc/init.d/salt-$fname ] && [ $_FORCE_OVERWRITE -eq $BS_TRUE ]); then - if [ -f "${_SALT_GIT_CHECKOUT_DIR}/debian/salt-$fname.init" ]; then - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/debian/salt-$fname.init" "/etc/init.d/salt-$fname" + # Install initscripts for Debian 7 "Wheezy" + elif [ ! -f /etc/init.d/salt-$fname ] || \ + ([ -f /etc/init.d/salt-$fname ] && [ $_FORCE_OVERWRITE -eq $BS_TRUE ]); then + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-$fname.init" ]; then + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.init" "/etc/init.d/salt-${fname}" + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.environment" "/etc/default/salt-${fname}" else - __fetch_url "/etc/init.d/salt-$fname" "${HTTP_VAL}://anonscm.debian.org/cgit/pkg-salt/salt.git/plain/debian/salt-${fname}.init" + __fetch_url "/etc/init.d/salt-${fname}" "${HTTP_VAL}://anonscm.debian.org/cgit/pkg-salt/salt.git/plain/debian/salt-${fname}.init" fi - if [ ! -f "/etc/init.d/salt-$fname" ]; then - echowarn "The init script for salt-$fname was not found, skipping it..." + + if [ ! -f "/etc/init.d/salt-${fname}" ]; then + echowarn "The init script for salt-${fname} was not found, skipping it..." continue fi - chmod +x "/etc/init.d/salt-$fname" + + chmod +x "/etc/init.d/salt-${fname}" # Skip salt-api since the service should be opt-in and not necessarily started on boot - [ $fname = "api" ] && continue + [ "$fname" = "api" ] && continue - update-rc.d "salt-$fname" defaults + update-rc.d "salt-${fname}" defaults fi done From 0474f4399d115d0cc69e9f2215ab4ca472799450 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Thu, 28 Apr 2016 10:49:39 +0300 Subject: [PATCH 4/8] Debian: check if the service is going to be started at any runlevel, fixes bootstrap in container (Docker, LXC) --- bootstrap-salt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index da3652c..dd73884 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -1889,8 +1889,8 @@ __check_services_debian() { servicename=$1 echodebug "Checking if service ${servicename} is enabled" - # shellcheck disable=SC2086,SC2046,SC2144 - if [ -f /etc/rc$(runlevel | awk '{ print $2 }').d/S*${servicename} ]; then + # Check if the service is going to be started at any runlevel, fixes bootstrap in container (Docker, LXC) + if ls /etc/rc?.d/S*"${servicename}" >/dev/null 2>&1; then echodebug "Service ${servicename} is enabled" return 0 else From 8291480da9367ab29fc1de7461e489b62bd27dbe Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Fri, 29 Apr 2016 17:03:48 +0300 Subject: [PATCH 5/8] Disable shell warning about unbound variable during git install --- bootstrap-salt.sh | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index dd73884..1c9bd1d 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -477,22 +477,26 @@ if [ "$ITYPE" = "git" ]; then GIT_REV="$1" shift fi + + # Disable shell warning about unbound variable during git install + STABLE_REV="" + # If doing stable install, check if version specified elif [ "$ITYPE" = "stable" ]; then if [ "$#" -eq 0 ];then STABLE_REV="latest" else __check_unparsed_options "$*" - if [ "$(echo "$1" | egrep '^(latest|1\.6|1\.7|2014\.1|2014\.7|2015\.5|2015\.8)$')" != "" ]; then - STABLE_REV="$1" - shift - elif [ "$(echo "$1" | egrep '^([0-9]*\.[0-9]*\.[0-9]*)$')" != "" ]; then - STABLE_REV="archive/$1" - shift - else - echo "Unknown stable version: $1 (valid: 1.6, 1.7, 2014.1, 2014.7, 2015.5, 2015.8, latest, \$MAJOR.\$MINOR.\$PATCH)" - exit 1 + if [ "$(echo "$1" | egrep '^(latest|1\.6|1\.7|2014\.1|2014\.7|2015\.5|2015\.8)$')" != "" ]; then + STABLE_REV="$1" + shift + elif [ "$(echo "$1" | egrep '^([0-9]*\.[0-9]*\.[0-9]*)$')" != "" ]; then + STABLE_REV="archive/$1" + shift + else + echo "Unknown stable version: $1 (valid: 1.6, 1.7, 2014.1, 2014.7, 2015.5, 2015.8, latest, \$MAJOR.\$MINOR.\$PATCH)" + exit 1 fi fi fi From a9da2c6102ee17a2b66cf7e291f8f399ba1678ae Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Sat, 30 Apr 2016 16:34:34 +0300 Subject: [PATCH 6/8] Drop repetitive commands in `install_debian_8_git_deps()` function which alredy have been executed in `install_debian_8_deps` --- bootstrap-salt.sh | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 1c9bd1d..f3050c0 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2956,18 +2956,11 @@ install_debian_7_git_deps() { install_debian_8_git_deps() { install_debian_8_deps || return 1 - # No user interaction, libc6 restart services for example - export DEBIAN_FRONTEND=noninteractive - if ! __check_command_exists git; then __apt_get_install_noinput git || return 1 fi - if [ "$(dpkg-query -l 'python-zmq')" = "" ]; then - __apt_get_install_noinput libzmq3 libzmq3-dev python-zmq || return 1 - fi - - __apt_get_install_noinput lsb-release python python-pkg-resources python-crypto \ + __apt_get_install_noinput lsb-release python-pkg-resources python-crypto \ python-jinja2 python-m2crypto python-yaml msgpack-python || return 1 __git_clone_and_checkout || return 1 @@ -2994,16 +2987,6 @@ install_debian_8_git_deps() { CONFIG_SALT_FUNC="config_salt" fi - if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then - __apt_get_upgrade_noinput || return 1 - fi - - if [ "${_EXTRA_PACKAGES}" != "" ]; then - echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" - # shellcheck disable=SC2086 - __apt_get_install_noinput ${_EXTRA_PACKAGES} || return 1 - fi - return 0 } From 70e1105404ab303ea4f0bb376958717a3deda9a3 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Sun, 1 May 2016 20:05:55 +0300 Subject: [PATCH 7/8] Debian 8: do git checkout before install majority of pkg dependencies --- bootstrap-salt.sh | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index f3050c0..9285752 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2960,27 +2960,32 @@ install_debian_8_git_deps() { __apt_get_install_noinput git || return 1 fi - __apt_get_install_noinput lsb-release python-pkg-resources python-crypto \ - python-jinja2 python-m2crypto python-yaml msgpack-python || return 1 - __git_clone_and_checkout || return 1 + __PACKAGES="lsb-release python-pkg-resources python-crypto python-jinja2 python-m2crypto python-yaml msgpack-python" + __PIP_PACKAGES="" + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" ]; then - __REQUIRED_TORNADO="$(grep tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt")" + __REQUIRED_TORNADO="$(grep ^tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" | tr -d ' ')" if [ -n "${__REQUIRED_TORNADO}" ]; then __check_pip_allowed "You need to allow pip based installations (-P) in order to install required 'tornado' module" - __PACKAGES="python-dev" + __PACKAGES="${__PACKAGES} python-dev" + __PIP_PACKAGES="${__PIP_PACKAGES} tornado" if ! __check_command_exists pip; then __PACKAGES="${__PACKAGES} python-pip" fi - - # shellcheck disable=SC2086 - __apt_get_install_noinput ${__PACKAGES} || return 1 - pip install -U tornado || return 1 fi fi + # shellcheck disable=SC2086 + __apt_get_install_noinput ${__PACKAGES} || return 1 + + if [ "${__PIP_PACKAGES}" != "" ]; then + # shellcheck disable=SC2086,SC2090 + pip install -U ${__PIP_PACKAGES} || return 1 + fi + # Let's trigger config_salt() if [ "$_TEMP_CONFIG_DIR" = "null" ]; then _TEMP_CONFIG_DIR="${_SALT_GIT_CHECKOUT_DIR}/conf/" From 29cad9c168131dab529c9b8a5dcb67dac3aad568 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Tue, 17 May 2016 15:53:34 +0300 Subject: [PATCH 8/8] Enable backports repo on Debian 8 if `pip` installation is not allowed --- bootstrap-salt.sh | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 9285752..15d0338 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2967,13 +2967,24 @@ install_debian_8_git_deps() { if [ -f "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" ]; then __REQUIRED_TORNADO="$(grep ^tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" | tr -d ' ')" - if [ -n "${__REQUIRED_TORNADO}" ]; then - __check_pip_allowed "You need to allow pip based installations (-P) in order to install required 'tornado' module" - __PACKAGES="${__PACKAGES} python-dev" - __PIP_PACKAGES="${__PIP_PACKAGES} tornado" - if ! __check_command_exists pip; then - __PACKAGES="${__PACKAGES} python-pip" + if [ -n "${__REQUIRED_TORNADO}" ]; then + if (__check_pip_allowed >/dev/null 2>&1); then + __PACKAGES="${__PACKAGES} python-dev" + __PIP_PACKAGES="${__PIP_PACKAGES} tornado" + + if ! __check_command_exists pip; then + __PACKAGES="${__PACKAGES} python-pip" + fi + else + # Check if Debian Backports repo already configured + if ! apt-cache policy | grep -q 'Debian Backports'; then + echo 'deb http://httpredir.debian.org/debian jessie-backports main' > \ + /etc/apt/sources.list.d/backports.list + apt-get update + fi + + __PACKAGES="${__PACKAGES} python-tornado/jessie-backports" fi fi fi