diff --git a/AUTHORS.rst b/AUTHORS.rst index 98e1618..02bd0e6 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -21,6 +21,7 @@ Angelo Gründler plueschopath angelo.gruendler@w1r3.net Ari Aosved devaos ari.aosved@gmail.com Ashok Raja R ashokrajar ashokrajar@users.noreply.github.com Beau Hargis beaucephus beau@customermobile.com +Benjamin Drung bdrung bdrung@debian.org BlaineAtAffirmOnceMore BlaineAtAffirm Boris Feld Lothiraldan Brad Thurber bradthurber @@ -70,6 +71,7 @@ Henrik Holmboe holmboe Howard Mei HowardMei howardleomei@gmail.com James Booth absolutejam vvalentine1337@gmail.com Jared E Stroud jaredestroud jaredestroud@gmail.com +Jasper Lievisse Adriaanse jasperla jasper@humppa.nl JD decomposite Jeff Hui jeffh jeff@jeffhui.net Jeff Strunk jstrunk @@ -112,6 +114,7 @@ Pedro Paulo pedropaulovc Pete Lumbis plumbis Peter Tripp notpeter Petr Michalec epcim +pjcreath pjcreath Prayag Verma pra85 prayag.verma@gmail.com ptonelli ptonelli Randy Thompson beardedeagle randy@heroictek.com @@ -131,6 +134,7 @@ Shawn Butts shawnbutts Skyler Berg skylerberg skylertheberg@gmail.com Stanislav B stanislavb Steve Groesz wolfpackmars2 wolfpackmars2@yahoo.com +sybix sybix Tate Eskew tateeskew Thomas S. Hatch thatch45 thatch45@saltstack.com Tobias Jungel toanju Tobias.Jungel@gmail.com diff --git a/ChangeLog b/ChangeLog index f5618a6..a396058 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,26 @@ +Version 2018.04.25: + * Install py-tornado4 for FreeBSD. (abednarik) #1219 + * Remove COPR repos configuration for Fedora (vutny ) #1211 + * Fix for silently ignored version argument on CentOS/RHEL (pjcreath) #1210 + * Use integer parameter for sleep command (bdrung) #1205 + * Add 2018.3 branch to list of stable options (rallytime) #1204 + * If installing with -P, install tornado<5.0. (rallytime) #1203 + * Add M2Crypto to python27 centos 6 bootstrap install (Ch3LL) #1201 + * Update README to include mention of bootstrap-salt.ps1 for Windows (rallytime) #1200 + * Port spelling fixes from change in Salt to Bootstrap (rallytime) #1199 + * Add M2Crypto package back to git install functions (rallytime) #1198 + * OpenBSD has a cdn which handles selecting the best mirror (jasperla) #1197 + * Change gnupg2 pacakge to gnupg for non-LTS versions of Ubuntu (rallytime) #1196 + * Install swig30 as freebsd dep instead of swig. (abednarik) #1191 + * make salt-syndic optional on salt bootstrap (sybix) #1190 + * Add __wait_for_apt function: avoid locking on the apt-get process (rallytime) #1186 + * Update the README.rst file with some grammatical changes (rallytime) #1185 + * Update Fedora support: 25 is EOL, 27 is supported (rallytime) #1184 + * Add a note about the use of sudo when running commands to README (rallytime) #1183 + * Adding support for minor release pinning on AWS Linux (cmclaughlin) #1182 + * Wait for zypper processes to finish before calling zypper again (rallytime) #1181 + * only install ca-certificates on opensuse if it isn't already installed (gtmanfred) #1179 + Version 2017.12.13: * Use HTTPS URL for OpenSuse's saltstack repo (gdm85) #1174 * Respect disable repos (-r) option on OpenBSD (eradman) #1171 diff --git a/README.rst b/README.rst index 37307ad..ae5ca3f 100644 --- a/README.rst +++ b/README.rst @@ -11,26 +11,28 @@ Before `Salt`_ can be used for provisioning on the desired machine, the binaries installed. Since `Salt`_ supports many different distributions and versions of operating systems, the `Salt`_ installation process is handled by this shell script ``bootstrap-salt.sh``. This script runs through a series of checks to determine operating system type and version to then -install the `Salt`_ binaries using the appropriate methods. +install the `Salt`_ binaries using the appropriate methods. For Windows, use the +``bootstrap-salt.ps1`` script. **NOTE** -This ``README`` file is not the absolute truth to what the bootstrap script is capable of, for +This ``README`` file is not the absolute truth as to what the bootstrap script is capable of. For that, please read the generated help by passing ``-h`` to the script or even better, `read the source`_. Bootstrap ========= -In every two-step installation example, you would be well-served to **verify against the SHA256 sum** -of the downloaded ``bootstrap-salt.sh`` file. +In every two-step installation example, you would be well-served to **verify against the SHA256 +sum** of the downloaded ``bootstrap-salt.sh`` file. The SHA256 sum of the ``bootstrap-salt.sh`` file, per release, is: +- 2017.12.13: ``c127b3aa4a8422f6b81f5b4a40d31d13cec97bf3a39bca9c11a28f24910a6895`` - 2017.08.17: ``909b4d35696b9867b34b22ef4b60edbc5a0e9f8d1ed8d05f922acb79a02e46e3`` - 2017.05.24: ``8c42c2e5ad3d4384ddc557da5c214ba3e40c056ca1b758d14a392c1364650e89`` -If you're looking for the *one-liner* to install Salt, please scroll to the bottom and use the +If you're looking for a *one-liner* to install Salt, please scroll to the bottom and use the instructions for `Installing via an Insecure One-Liner`_. Contributing @@ -43,11 +45,12 @@ Examples -------- The Salt Bootstrap script has a wide variety of options that can be passed as -well as several ways of obtaining the bootstrap script itself. +well as several ways of obtaining the bootstrap script itself. Note that the use of ``sudo`` +is not needed when running these commands as the ``root`` user. **NOTE** -These examples below show how to bootstrap Salt directly from GitHub or other Git repository. +The examples below show how to bootstrap Salt directly from GitHub or another Git repository. Run the script without any parameters to get latest stable Salt packages for your system from `SaltStack's corporate repository`_. See first example in the `Install using wget`_ section. @@ -111,14 +114,14 @@ Installing a specific version from git using ``wget``: **NOTE** -On the above example we added `-P` which will allow PIP packages to be installed if required but -it's not a necessary flag for Git based bootstraps. +On the above example we added ``-P`` which will allow PIP packages to be installed if required. +However, the ``-P`` flag is not necessary for Git-based bootstraps. Install using Python ~~~~~~~~~~~~~~~~~~~~ -If you already have Python installed, ``python 2.6``, then it's as easy as: +If you already have Python installed, ``python 2.7``, then it's as easy as: .. code:: console @@ -136,7 +139,7 @@ All Python versions should support the following in-line code: Install using fetch ~~~~~~~~~~~~~~~~~~~ -On a FreeBSD base system you usually don't have either of the above binaries available. You **do** +On a FreeBSD-based system you usually don't have either of the above binaries available. You **do** have ``fetch`` available though: .. code:: console @@ -173,7 +176,7 @@ The following examples illustrate how to install Salt via a one-liner. Warning! These methods do not involve a verification step and assume that the delivered file is trustworthy. -Any of the example above which use two-lines can be made to run in a single-line +Any of the examples above which use two lines can be made to run in a single-line configuration with minor modifications. Installing the latest stable release of Salt (default): @@ -230,8 +233,8 @@ Debian and derivatives - Cumulus Linux 2/3 - Debian GNU/Linux 7/8/9 - Devuan GNU/Linux 1/2 -- Linux Mint Debian Edition 1 (based on Debian 8) - Kali Linux 1.0 (based on Debian 7) +- Linux Mint Debian Edition 1 (based on Debian 8) - Raspbian 8 (``armhf`` packages) and 9 (using ``git`` installation mode only) Debian Best Effort Support: Testing Release @@ -253,7 +256,7 @@ Red Hat family - Amazon Linux 2012.3 and later - CentOS 6/7 - Cloud Linux 6/7 -- Fedora 25/26 +- Fedora 26/27/28 (install latest stable from standard repositories) - Oracle Linux 6/7 - Red Hat Enterprise Linux 6/7 - Scientific Linux 6/7 @@ -274,20 +277,20 @@ Ubuntu and derivatives - Linux Mint 17/18 - Ubuntu 14.04/16.04 and subsequent non-TLS releases (see below) -Ubuntu Best Effort Support: Non-LTS Releases +Ubuntu Best Effort Support: Non-LTS Releases ******************************************** -This script provides best-effort support for current, non-LTS Ubuntu releases. If package +This script provides best-effort support for current, non-LTS Ubuntu releases. If package repositories are not provided on `SaltStack's Ubuntu repository`_ for the non-LTS release, the bootstrap script will attempt to install the packages for the most closely related LTS Ubuntu release instead. -For example, when installing Salt on Ubuntu 16.10, the bootstrap script will setup the repository +For example, when installing Salt on Ubuntu 17.10, the bootstrap script will setup the repository for Ubuntu 16.04 from `SaltStack's Ubuntu repository`_ and install the 16.04 packages. -Other Linux distro -~~~~~~~~~~~~~~~~~~ +Other Linux distributions +~~~~~~~~~~~~~~~~~~~~~~~~~ - Alpine Linux 3.5/edge - Arch Linux @@ -309,27 +312,27 @@ UNIX systems Unsupported Distro ------------------ -You found a Linux distribution which we still do not support or we do not correctly identify? -Please run the following commands and report their output when creating a ticket: +If you are running a Linux distribution that is not supported yet or is not correctly identified, +please run the following commands and report their output when creating an issue: .. code:: console sudo find /etc/ -name \*-release -print -exec cat {} \; command lsb_release -a -For information on how to add support for a currently unsupported distro, please refer to the +For information on how to add support for a currently unsupported distribution, please refer to the `Contributing Guidelines`_. Testing ------- -There are a couple of ways to test the bootstrap script. Running the script on a full-fledged +There are a couple of ways to test the bootstrap script. Running the script on a fully-fledged VM is one way. Other options include using Vagrant or Docker. Testing in Vagrant ================== -You can use Vagrant_ to easily test changes on a clean machine. The ``Vagrantfile`` defaults to an +Vagrant_ can be used to easily test changes on a clean machine. The ``Vagrantfile`` defaults to an Ubuntu box. First, install Vagrant, then: .. code:: console @@ -340,9 +343,9 @@ Ubuntu box. First, install Vagrant, then: Running in Docker ================= -Also you are able to run and use Salt inside Docker_ container on Linux machine. -Let's prepare the Docker image using provided ``Dockerfile`` to install both Salt Master and Minion -with the bootstrap script: +It is possible to run and use Salt inside a Docker_ container on Linux machines. +Let's prepare the Docker image using the provided ``Dockerfile`` to install both a Salt Master +and a Salt Minion with the bootstrap script: .. code:: console @@ -361,17 +364,17 @@ And finally "enter" the running container and make Salt fully operational: docker exec -i -t salt /bin/bash salt-key -A -y -Salt is ready and working in the Docker container with Minion authenticated on Master. +Salt is ready and working in the Docker container with the Minion authenticated on the Master. **NOTE** -The ``Dockerfile`` here inherits Ubuntu 14.04 public image with Upstart configured as init system. -Consider it as an example or starting point of how to make your own Docker images with suitable -Salt components, custom configurations and even `pre-accepted Minion key`_ already installed. +The ``Dockerfile`` here inherits the Ubuntu 14.04 public image with Upstart configured as the init +system. Use it as an example or starting point of how to make your own Docker images with suitable +Salt components, custom configurations, and even `pre-accepted Minion keys`_ already installed. .. _Contributing Guidelines: https://github.com/saltstack/salt-bootstrap/blob/develop/CONTRIBUTING.md .. _Docker: https://www.docker.com/ -.. _`pre-accepted Minion key`: https://docs.saltstack.com/en/latest/topics/tutorials/preseed_key.html +.. _`pre-accepted Minion keys`: https://docs.saltstack.com/en/latest/topics/tutorials/preseed_key.html .. _`read the source`: https://github.com/saltstack/salt-bootstrap/blob/develop/bootstrap-salt.sh .. _`Salt`: https://saltstack.com/community/ .. _`Salt's Supported Operating Systems`: http://saltstack.com/wp-content/uploads/2016/08/SaltStack-Supported-Operating-Systems.pdf diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 9280139..f0727c6 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -9,7 +9,7 @@ # # BUGS: https://github.com/saltstack/salt-bootstrap/issues # -# COPYRIGHT: (c) 2012-2017 by the SaltStack Team, see AUTHORS.rst for more +# COPYRIGHT: (c) 2012-2018 by the SaltStack Team, see AUTHORS.rst for more # details. # # LICENSE: Apache 2.0 @@ -18,7 +18,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2017.12.13" +__ScriptVersion="2018.04.25" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -95,7 +95,7 @@ echoinfo() { #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: echowarn -# DESCRIPTION: Echo warning informations to stdout. +# DESCRIPTION: Echo warning information to stdout. #---------------------------------------------------------------------------------------------------------------------- echowarn() { printf "${YC} * WARN${EC}: %s\n" "$@"; @@ -249,7 +249,6 @@ _CURL_ARGS=${BS_CURL_ARGS:-} _FETCH_ARGS=${BS_FETCH_ARGS:-} _GPG_ARGS=${BS_GPG_ARGS:-} _WGET_ARGS=${BS_WGET_ARGS:-} -_ENABLE_EXTERNAL_ZMQ_REPOS=${BS_ENABLE_EXTERNAL_ZMQ_REPOS:-$BS_FALSE} _SALT_MASTER_ADDRESS=${BS_SALT_MASTER_ADDRESS:-null} _SALT_MINION_ID="null" # _SIMPLIFY_VERSION is mostly used in Solaris based distributions @@ -299,13 +298,13 @@ __usage() { Examples: - ${__ScriptName} - ${__ScriptName} stable - - ${__ScriptName} stable 2016.3 - - ${__ScriptName} stable 2016.3.1 + - ${__ScriptName} stable 2017.7 + - ${__ScriptName} stable 2017.7.2 - ${__ScriptName} daily - ${__ScriptName} testing - ${__ScriptName} git - - ${__ScriptName} git 2016.3 - - ${__ScriptName} git v2016.3.1 + - ${__ScriptName} git 2017.7 + - ${__ScriptName} git v2017.7.2 - ${__ScriptName} git 06f249901a2e2f1ed310d58ea3921a129f214358 Options: @@ -338,7 +337,7 @@ __usage() { -U If set, fully upgrade the system prior to bootstrapping Salt -I If set, allow insecure connections while downloading any files. For example, pass '--no-check-certificate' to 'wget' or '--insecure' to - 'curl'. On Debian and Ubuntu, using this option with -U allows to obtain + 'curl'. On Debian and Ubuntu, using this option with -U allows obtaining GnuPG archive keys insecurely if distro has changed release signatures. -F Allow copied files to overwrite existing (config, init.d, etc) -K If set, keep the temporary files in the temporary directories specified @@ -355,8 +354,6 @@ __usage() { per -p flag. You're responsible for providing the proper package name. -H Use the specified HTTP proxy for all download URLs (including https://). For example: http://myproxy.example.com:3128 - -Z Enable additional package repository for newer ZeroMQ - (only available for RHEL/CentOS/Fedora/Ubuntu based distributions) -b Assume that dependencies are already installed and software sources are set up. If git is selected, git tree is still checked out as dependency step. @@ -395,7 +392,7 @@ __usage() { tested with Centos 6 and is considered experimental. This will install the ius repo on the box if disable repo is false. This must be used in conjunction with -x . For example: - sh bootstrap.sh -P -y -x python2.7 git v2016.11.3 + sh bootstrap.sh -P -y -x python2.7 git v2017.7.2 The above will install python27 and install the git version of salt using the python2.7 executable. This only works for git and pip installations. @@ -438,7 +435,6 @@ do p ) _EXTRA_PACKAGES="$_EXTRA_PACKAGES $OPTARG" ;; d ) _DISABLE_SALT_CHECKS=$BS_TRUE ;; H ) _HTTP_PROXY="$OPTARG" ;; - Z ) _ENABLE_EXTERNAL_ZMQ_REPOS=$BS_TRUE ;; b ) _NO_DEPS=$BS_TRUE ;; f ) _FORCE_SHALLOW_CLONE=$BS_TRUE ;; l ) _DISABLE_SSL=$BS_TRUE ;; @@ -593,14 +589,14 @@ elif [ "$ITYPE" = "stable" ]; then if [ "$#" -eq 0 ];then STABLE_REV="latest" else - if [ "$(echo "$1" | egrep '^(latest|1\.6|1\.7|2014\.1|2014\.7|2015\.5|2015\.8|2016\.3|2016\.11|2017\.7)$')" != "" ]; then + if [ "$(echo "$1" | egrep '^(latest|1\.6|1\.7|2014\.1|2014\.7|2015\.5|2015\.8|2016\.3|2016\.11|2017\.7|2018\.3)$')" != "" ]; then STABLE_REV="$1" shift elif [ "$(echo "$1" | egrep '^([0-9]*\.[0-9]*\.[0-9]*)$')" != "" ]; then STABLE_REV="archive/$1" shift else - echo "Unknown stable version: $1 (valid: 1.6, 1.7, 2014.1, 2014.7, 2015.5, 2015.8, 2016.3, 2016.11, 2017.7, latest, \$MAJOR.\$MINOR.\$PATCH)" + echo "Unknown stable version: $1 (valid: 1.6, 1.7, 2014.1, 2014.7, 2015.5, 2015.8, 2016.3, 2016.11, 2017.7, 2018.3, latest, \$MAJOR.\$MINOR.\$PATCH)" exit 1 fi fi @@ -1331,10 +1327,10 @@ __check_dpkg_architecture() { if [ "${error_msg}" != "" ]; then echoerror "${error_msg}" if [ "$ITYPE" != "git" ]; then - echoerror "You can try git installation mode, i.e.: sh ${__ScriptName} git v2016.11.5." + echoerror "You can try git installation mode, i.e.: sh ${__ScriptName} git v2017.7.2." echoerror "It may be necessary to use git installation mode with pip and disable the SaltStack apt repository." echoerror "For example:" - echoerror " sh ${__ScriptName} -r -P git v2016.11.5" + echoerror " sh ${__ScriptName} -r -P git v2017.7.2" fi fi @@ -1372,16 +1368,10 @@ __ubuntu_codename_translation() { DISTRO_CODENAME="trusty" ;; "16") - if [ "$_april" ]; then - DISTRO_CODENAME="xenial" - else - DISTRO_CODENAME="yakkety" - fi + DISTRO_CODENAME="xenial" ;; "17") - if [ "$_april" ]; then - DISTRO_CODENAME="zesty" - fi + DISTRO_CODENAME="artful" ;; *) DISTRO_CODENAME="trusty" @@ -1500,9 +1490,12 @@ __check_end_of_life_versions() { # < 14.04 # = 14.10 # = 15.04, 15.10 + # = 16.10 + # = 17.04 if [ "$DISTRO_MAJOR_VERSION" -lt 14 ] || \ [ "$DISTRO_MAJOR_VERSION" -eq 15 ] || \ - ([ "$DISTRO_MAJOR_VERSION" -lt 16 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]); then + ([ "$DISTRO_MAJOR_VERSION" -eq 17 ] && [ "$DISTRO_MINOR_VERSION" -eq 04 ]) || \ + ([ "$DISTRO_MAJOR_VERSION" -lt 17 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]); then echoerror "End of life distributions are not supported." echoerror "Please consider upgrading to the next stable. See:" echoerror " https://wiki.ubuntu.com/Releases" @@ -1544,8 +1537,8 @@ __check_end_of_life_versions() { ;; fedora) - # Fedora lower than 25 are no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 25 ]; then + # Fedora lower than 26 are no longer supported + if [ "$DISTRO_MAJOR_VERSION" -lt 26 ]; then echoerror "End of life distributions are not supported." echoerror "Please consider upgrading to the next stable. See:" echoerror " https://fedoraproject.org/wiki/Releases" @@ -1765,12 +1758,41 @@ __function_defined() { } +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __wait_for_apt +# DESCRIPTION: Check if any apt, apt-get, aptitude, or dpkg processes are running before +# calling these again. This is useful when these process calls are part of +# a boot process, such as on AWS AMIs. This func will wait until the boot +# process is finished so the script doesn't exit on a locked proc. +#---------------------------------------------------------------------------------------------------------------------- +__wait_for_apt(){ + echodebug "Checking if apt process is currently running." + + # Timeout set at 15 minutes + WAIT_TIMEOUT=900 + + while ps -C apt,apt-get,aptitude,dpkg >/dev/null; do + sleep 1 + WAIT_TIMEOUT=$((WAIT_TIMEOUT - 1)) + + # If timeout reaches 0, abort. + if [ "$WAIT_TIMEOUT" -eq 0 ]; then + echoerror "Apt, apt-get, aptitude, or dpkg process is taking too long." + echoerror "Bootstrap script cannot proceed. Aborting." + return 1 + fi + done + + echodebug "No apt processes are currently running." +} + #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __apt_get_install_noinput # DESCRIPTION: (DRY) apt-get install with noinput options # PARAMETERS: packages #---------------------------------------------------------------------------------------------------------------------- __apt_get_install_noinput() { + __wait_for_apt apt-get install -y -o DPkg::Options::=--force-confold "${@}"; return $? } # ---------- end of function __apt_get_install_noinput ---------- @@ -1780,6 +1802,7 @@ __apt_get_install_noinput() { # DESCRIPTION: (DRY) apt-get upgrade with noinput options #---------------------------------------------------------------------------------------------------------------------- __apt_get_upgrade_noinput() { + __wait_for_apt apt-get upgrade -y -o DPkg::Options::=--force-confold; return $? } # ---------- end of function __apt_get_upgrade_noinput ---------- @@ -1790,6 +1813,7 @@ __apt_get_upgrade_noinput() { # PARAMETERS: url #---------------------------------------------------------------------------------------------------------------------- __apt_key_fetch() { + __wait_for_apt url=$1 # shellcheck disable=SC2086 @@ -2544,7 +2568,7 @@ __enable_universe_repository() { __install_saltstack_ubuntu_repository() { # Workaround for latest non-LTS ubuntu - if [ "$DISTRO_VERSION" = "16.10" ] || [ "$DISTRO_MAJOR_VERSION" -gt 16 ]; then + if [ "$DISTRO_MAJOR_VERSION" -gt 16 ]; then echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages from latest LTS release. You may experience problems." UBUNTU_VERSION=16.04 UBUNTU_CODENAME="xenial" @@ -2556,8 +2580,8 @@ __install_saltstack_ubuntu_repository() { __PACKAGES='' # Install downloader backend for GPG keys fetching - if [ "$DISTRO_VERSION" = "16.10" ] || [ "$DISTRO_MAJOR_VERSION" -gt 16 ]; then - __PACKAGES="${__PACKAGES} gnupg2 dirmngr" + if [ "$DISTRO_MAJOR_VERSION" -gt 16 ]; then + __PACKAGES="${__PACKAGES} gnupg dirmngr" else __PACKAGES="${__PACKAGES} gnupg-curl" fi @@ -2576,6 +2600,7 @@ __install_saltstack_ubuntu_repository() { __apt_key_fetch "$SALTSTACK_UBUNTU_URL/SALTSTACK-GPG-KEY.pub" || return 1 + __wait_for_apt apt-get update } @@ -2588,6 +2613,7 @@ install_ubuntu_deps() { __enable_universe_repository || return 1 + __wait_for_apt apt-get update fi @@ -2644,6 +2670,7 @@ install_ubuntu_stable_deps() { # No user interaction, libc6 restart services for example export DEBIAN_FRONTEND=noninteractive + __wait_for_apt apt-get update if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then @@ -2664,6 +2691,7 @@ install_ubuntu_stable_deps() { } install_ubuntu_daily_deps() { + __wait_for_apt install_ubuntu_stable_deps || return 1 if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then @@ -2681,6 +2709,7 @@ install_ubuntu_daily_deps() { } install_ubuntu_git_deps() { + __wait_for_apt apt-get update if ! __check_command_exists git; then @@ -2711,8 +2740,8 @@ install_ubuntu_git_deps() { else install_ubuntu_stable_deps || return 1 - __PACKAGES="${__PACKAGES} python-crypto python-jinja2 python-msgpack python-requests" - __PACKAGES="${__PACKAGES} python-tornado python-yaml python-zmq" + __PACKAGES="${__PACKAGES} python-crypto python-jinja2 python-m2crypto python-msgpack" + __PACKAGES="${__PACKAGES} python-requests python-tornado python-yaml python-zmq" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then # Install python-libcloud if asked to @@ -2791,7 +2820,7 @@ install_ubuntu_stable_post() { /bin/systemctl preset salt-$fname.service > /dev/null 2>&1 && /bin/systemctl enable salt-$fname.service > /dev/null 2>&1 ) - sleep 0.1 + sleep 1 /bin/systemctl daemon-reload elif [ -f /etc/init.d/salt-$fname ]; then update-rc.d salt-$fname defaults @@ -2817,7 +2846,7 @@ install_ubuntu_git_post() { [ $fname = "api" ] && continue systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) - sleep 0.1 + sleep 1 systemctl daemon-reload elif [ -f /sbin/initctl ]; then _upstart_conf="/etc/init/salt-$fname.conf" @@ -2973,6 +3002,7 @@ __install_saltstack_debian_repository() { __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALTSTACK-GPG-KEY.pub" || return 1 + __wait_for_apt apt-get update } @@ -2984,6 +3014,7 @@ install_debian_deps() { # No user interaction, libc6 restart services for example export DEBIAN_FRONTEND=noninteractive + __wait_for_apt apt-get update if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then @@ -3030,9 +3061,9 @@ install_debian_git_deps() { __git_clone_and_checkout || return 1 - __PACKAGES="libzmq3 libzmq3-dev lsb-release python-apt python-backports.ssl-match-hostname python-crypto" - __PACKAGES="${__PACKAGES} python-jinja2 python-msgpack python-requests" - __PACKAGES="${__PACKAGES} python-tornado python-yaml python-zmq" + __PACKAGES="libzmq3 libzmq3-dev lsb-release python-apt python-backports.ssl-match-hostname" + __PACKAGES="${__PACKAGES} python-crypto python-jinja2 python-msgpack python-m2crypto" + __PACKAGES="${__PACKAGES} python-requests python-tornado python-yaml python-zmq" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then # Install python-libcloud if asked to @@ -3071,8 +3102,9 @@ install_debian_8_git_deps() { __git_clone_and_checkout || return 1 - __PACKAGES="libzmq3 libzmq3-dev lsb-release python-apt python-crypto python-jinja2 python-msgpack" - __PACKAGES="${__PACKAGES} python-requests python-systemd python-yaml python-zmq" + __PACKAGES="libzmq3 libzmq3-dev lsb-release python-apt python-crypto python-jinja2" + __PACKAGES="${__PACKAGES} python-m2crypto python-msgpack python-requests python-systemd" + __PACKAGES="${__PACKAGES} python-yaml python-zmq" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then # Install python-libcloud if asked to @@ -3081,7 +3113,7 @@ install_debian_8_git_deps() { __PIP_PACKAGES='' if (__check_pip_allowed >/dev/null 2>&1); then - __PIP_PACKAGES='tornado' + __PIP_PACKAGES='tornado<5.0' # Install development environment for building tornado Python module __PACKAGES="${__PACKAGES} build-essential python-dev" @@ -3096,6 +3128,7 @@ install_debian_8_git_deps() { /etc/apt/sources.list.d/backports.list fi + __wait_for_apt apt-get update || return 1 # python-tornado package should be installed from backports repo @@ -3135,8 +3168,8 @@ install_debian_9_git_deps() { __git_clone_and_checkout || return 1 __PACKAGES="libzmq5 lsb-release python-apt python-backports-abc python-crypto" - __PACKAGES="${__PACKAGES} python-jinja2 python-msgpack python-requests python-systemd" - __PACKAGES="${__PACKAGES} python-tornado python-yaml python-zmq" + __PACKAGES="${__PACKAGES} python-jinja2 python-m2crypto python-msgpack python-requests" + __PACKAGES="${__PACKAGES} python-systemd python-tornado python-yaml python-zmq" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then # Install python-libcloud if asked to @@ -3330,15 +3363,8 @@ install_debian_check_services() { install_fedora_deps() { - if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then - if [ "$_ENABLE_EXTERNAL_ZMQ_REPOS" -eq $BS_TRUE ]; then - __install_saltstack_copr_zeromq_repository || return 1 - fi - - __install_saltstack_copr_salt_repository || return 1 - fi - - __PACKAGES="PyYAML libyaml python-crypto python-jinja2 python-zmq python2-msgpack python2-requests" + __PACKAGES="libyaml m2crypto PyYAML python-crypto python-jinja2" + __PACKAGES="${__PACKAGES} python2-msgpack python2-requests python-zmq" if [ "$DISTRO_MAJOR_VERSION" -lt 26 ]; then __PACKAGES="${__PACKAGES} yum-utils" @@ -3395,7 +3421,7 @@ install_fedora_stable_post() { [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) - sleep 0.1 + sleep 1 systemctl daemon-reload done } @@ -3456,7 +3482,7 @@ install_fedora_git_post() { [ $fname = "api" ] && continue systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) - sleep 0.1 + sleep 1 systemctl daemon-reload done } @@ -3523,20 +3549,6 @@ __install_epel_repository() { return 0 } -__install_saltstack_copr_zeromq_repository() { - echoinfo "Installing Zeromq >=4 and PyZMQ>=14 from SaltStack's COPR repository" - if [ ! -s /etc/yum.repos.d/saltstack-zeromq4.repo ]; then - if [ "${DISTRO_NAME_L}" = "fedora" ]; then - __REPOTYPE="${DISTRO_NAME_L}" - else - __REPOTYPE="epel" - fi - __fetch_url /etc/yum.repos.d/saltstack-zeromq4.repo \ - "${HTTP_VAL}://copr.fedorainfracloud.org/coprs/saltstack/zeromq4/repo/${__REPOTYPE}-${DISTRO_MAJOR_VERSION}/saltstack-zeromq4-${__REPOTYPE}-${DISTRO_MAJOR_VERSION}.repo" || return 1 - fi - return 0 -} - __install_saltstack_rhel_repository() { if [ "$ITYPE" = "stable" ]; then repo_rev="$STABLE_REV" @@ -3573,26 +3585,6 @@ _eof return 0 } -__install_saltstack_copr_salt_repository() { - echoinfo "Adding SaltStack's COPR repository" - - if [ "${DISTRO_NAME_L}" = "fedora" ]; then - [ "$DISTRO_MAJOR_VERSION" -ge 22 ] && return 0 - __REPOTYPE="${DISTRO_NAME_L}" - else - __REPOTYPE="epel" - fi - - __REPO_FILENAME="saltstack-salt-${__REPOTYPE}-${DISTRO_MAJOR_VERSION}.repo" - - if [ ! -s "/etc/yum.repos.d/${__REPO_FILENAME}" ]; then - __fetch_url "/etc/yum.repos.d/${__REPO_FILENAME}" \ - "${HTTP_VAL}://copr.fedorainfracloud.org/coprs/saltstack/salt/repo/${__REPOTYPE}-${DISTRO_MAJOR_VERSION}/${__REPO_FILENAME}" || return 1 - fi - - return 0 -} - install_centos_stable_deps() { if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then yum -y update || return 1 @@ -3692,7 +3684,8 @@ install_centos_git_deps() { __git_clone_and_checkout || return 1 - __PACKAGES="python-crypto python-futures python-msgpack python-zmq python-jinja2 python-requests python-tornado" + __PACKAGES="m2crypto python-crypto python-futures python-jinja2 python-msgpack" + __PACKAGES="${__PACKAGES} python-requests python-tornado python-zmq" if [ "$DISTRO_MAJOR_VERSION" -ge 7 ]; then __PACKAGES="${__PACKAGES} systemd-python" @@ -3709,7 +3702,12 @@ install_centos_git_deps() { if [ "${_PY_EXE}" != "" ]; then # If "-x" is defined, install dependencies with pip based on the Python version given. - _PIP_PACKAGES="jinja2 msgpack-python pycrypto PyYAML tornado zmq" + _PIP_PACKAGES="m2crypto jinja2 msgpack-python pycrypto PyYAML tornado<5.0 zmq" + + # install swig and openssl on cent6 + if [ "$DISTRO_MAJOR_VERSION" -eq 6 ]; then + __yum_install_noinput openssl-devel swig || return 1 + fi if [ -f "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" ]; then for SINGLE_PACKAGE in $_PIP_PACKAGES; do @@ -4279,7 +4277,7 @@ install_alpine_linux_stable_deps() { install_alpine_linux_git_deps() { install_alpine_linux_stable_deps || return 1 - apk -U add python2 py-virtualenv py2-crypto py2-setuptools \ + apk -U add python2 py-virtualenv py2-crypto py2-m2crypto py2-setuptools \ py2-jinja2 py2-yaml py2-markupsafe py2-msgpack py2-psutil \ py2-zmq zeromq py2-requests || return 1 @@ -4371,6 +4369,7 @@ install_alpine_linux_restart_daemons() { # Skip if not meant to be installed [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue # Disable stdin to fix shell session hang on killing tee pipe /sbin/rc-service salt-$fname stop < /dev/null > /dev/null 2>&1 @@ -4386,6 +4385,7 @@ install_alpine_linux_check_services() { # Skip if not meant to be installed [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue __check_services_alpine salt-$fname || return 1 done @@ -4404,6 +4404,7 @@ daemons_running_alpine_linux() { # Skip if not meant to be installed [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue # shellcheck disable=SC2009 if [ "$(ps wwwaux | grep -v grep | grep salt-$fname)" = "" ]; then @@ -4431,10 +4432,20 @@ install_amazon_linux_ami_deps() { _USEAWS=$BS_FALSE pkg_append="python" - repo_rev="$(echo "${STABLE_REV}" | sed 's|.*\/||g')" + if [ "$ITYPE" = "stable" ]; then + repo_rev="$STABLE_REV" + else + repo_rev="latest" + fi + + if echo $repo_rev | egrep -q '^archive'; then + year=$(echo "$repo_rev" | cut -d '/' -f 2 | cut -c1-4) + else + year=$(echo "$repo_rev" | cut -c1-4) + fi if echo "$repo_rev" | egrep -q '^(latest|2016\.11)$' || \ - [ "$(echo "$repo_rev" | cut -c1-4)" -gt 2016 ]; then + [ "$year" -gt 2016 ]; then _USEAWS=$BS_TRUE pkg_append="python27" fi @@ -4481,7 +4492,8 @@ _eof # Package python-ordereddict-1.1-2.el6.noarch is obsoleted by python26-2.6.9-2.88.amzn1.x86_64 # which is already installed - __PACKAGES="${pkg_append}-PyYAML ${pkg_append}-crypto ${pkg_append}-msgpack ${pkg_append}-zmq ${pkg_append}-jinja2 ${pkg_append}-requests" + __PACKAGES="m2crypto ${pkg_append}-crypto ${pkg_append}-jinja2 ${pkg_append}-PyYAML" + __PACKAGES="${__PACKAGES} ${pkg_append}-msgpack ${pkg_append}-requests ${pkg_append}-zmq" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4634,7 +4646,7 @@ install_arch_linux_git_deps() { fi pacman -R --noconfirm python2-distribute pacman -Su --noconfirm --needed python2-crypto python2-setuptools python2-jinja \ - python2-markupsafe python2-msgpack python2-psutil \ + python2-m2crypto python2-markupsafe python2-msgpack python2-psutil \ python2-pyzmq zeromq python2-requests python2-systemd || return 1 __git_clone_and_checkout || return 1 @@ -4708,7 +4720,7 @@ install_arch_linux_post() { /usr/bin/systemctl preset salt-$fname.service > /dev/null 2>&1 && /usr/bin/systemctl enable salt-$fname.service > /dev/null 2>&1 ) - sleep 0.1 + sleep 1 /usr/bin/systemctl daemon-reload continue fi @@ -4736,7 +4748,7 @@ install_arch_linux_git_post() { /usr/bin/systemctl preset salt-${fname}.service > /dev/null 2>&1 && /usr/bin/systemctl enable salt-${fname}.service > /dev/null 2>&1 ) - sleep 0.1 + sleep 1 /usr/bin/systemctl daemon-reload continue fi @@ -4889,9 +4901,9 @@ install_freebsd_9_stable_deps() { __configure_freebsd_pkg_details || return 1 fi - # Now install swig + # Now install swig30 # shellcheck disable=SC2086 - /usr/local/sbin/pkg install ${FROM_FREEBSD} -y swig || return 1 + /usr/local/sbin/pkg install ${FROM_FREEBSD} -y swig30 || return 1 # YAML module is used for generating custom master/minion configs # shellcheck disable=SC2086 @@ -4938,7 +4950,7 @@ install_freebsd_git_deps() { # We're on the develop branch, install whichever tornado is on the requirements file __REQUIRED_TORNADO="$(grep tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt")" if [ "${__REQUIRED_TORNADO}" != "" ]; then - /usr/local/sbin/pkg install -y www/py-tornado || return 1 + /usr/local/sbin/pkg install -y www/py-tornado4 || return 1 fi fi @@ -5102,35 +5114,11 @@ install_freebsd_restart_daemons() { # OpenBSD Install Functions # -__choose_openbsd_mirror() { - OPENBSD_REPO='' - MINTIME='' - MIRROR_LIST=$(ftp -w 15 -Vao - 'https://ftp.openbsd.org/cgi-bin/ftplist.cgi?dbversion=1' | awk '/^http/ {print $1}') - - for MIRROR in $MIRROR_LIST; do - MIRROR_HOST=$(echo "$MIRROR" | sed -e 's|.*//||' -e 's|+*/.*$||') - TIME=$(ping -c 1 -w 1 -q "$MIRROR_HOST" | awk -F/ '/round-trip/ { print $5 }') - [ -z "$TIME" ] && continue - - echodebug "ping time for $MIRROR_HOST is $TIME" - if [ -z "$MINTIME" ]; then - FASTER_MIRROR=1 - else - FASTER_MIRROR=$(echo "$TIME < $MINTIME" | bc) - fi - if [ "$FASTER_MIRROR" -eq 1 ]; then - MINTIME=$TIME - OPENBSD_REPO="$MIRROR" - fi - done -} - install_openbsd_deps() { if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then - __choose_openbsd_mirror || return 1 - echoinfo "setting package repository to $OPENBSD_REPO with ping time of $MINTIME" - [ -n "$OPENBSD_REPO" ] || return 1 - echo "${OPENBSD_REPO}" >>/etc/installurl || return 1 + OPENBSD_REPO='https://cdn.openbsd.org/pub/OpenBSD' + echoinfo "setting package repository to $OPENBSD_REPO" + echo "${OPENBSD_REPO}" >/etc/installurl || return 1 fi if [ "${_EXTRA_PACKAGES}" != "" ]; then @@ -5230,7 +5218,7 @@ install_openbsd_restart_daemons() { # SmartOS Install Functions # install_smartos_deps() { - pkgin -y install zeromq py27-crypto py27-msgpack py27-yaml py27-jinja2 py27-zmq py27-requests || return 1 + pkgin -y install zeromq py27-crypto py27-m2crypto py27-msgpack py27-yaml py27-jinja2 py27-zmq py27-requests || return 1 # Set _SALT_ETC_DIR to SmartOS default if they didn't specify _SALT_ETC_DIR=${BS_SALT_ETC_DIR:-/opt/local/etc/salt} @@ -5460,6 +5448,13 @@ __version_lte() { } __zypper() { + # Check if any zypper process is running before calling zypper again. + # This is useful when a zypper call is part of a boot process and will + # wait until the zypper process is finished, such as on AWS AMIs. + while pgrep -l zypper; do + sleep 1 + done + zypper --non-interactive "${@}"; return $? } @@ -5519,7 +5514,7 @@ install_opensuse_stable_deps() { } install_opensuse_git_deps() { - if [ "$_INSECURE_DL" -eq $BS_FALSE ] && [ "${_SALT_REPO_URL%%://*}" = "https" ]; then + if [ "$_INSECURE_DL" -eq $BS_FALSE ] && [ "${_SALT_REPO_URL%%://*}" = "https" ] && ! __check_command_exists update-ca-certificates; then __zypper_install ca-certificates || return 1 fi @@ -5533,7 +5528,7 @@ install_opensuse_git_deps() { __git_clone_and_checkout || return 1 - __PACKAGES="libzmq5 python-Jinja2 python-msgpack-python python-pycrypto python-pyzmq python-xml" + __PACKAGES="libzmq5 python-Jinja2 python-m2crypto python-msgpack-python python-pycrypto python-pyzmq python-xml" if [ -f "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" ]; then # We're on the develop branch, install whichever tornado is on the requirements file @@ -5598,7 +5593,7 @@ install_opensuse_stable_post() { if [ -f /bin/systemctl ]; then systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) - sleep 0.1 + sleep 1 systemctl daemon-reload continue fi @@ -5727,6 +5722,12 @@ install_suse_12_stable_deps() { # shellcheck disable=SC2086,SC2090 __zypper_install ${__PACKAGES} || return 1 + # SLES 11 SP3 ships with both python-M2Crypto-0.22.* and python-m2crypto-0.21 and we will be asked which + # we want to install, even with --non-interactive. + # Let's try to install the higher version first and then the lower one in case of failure + __zypper_install 'python-M2Crypto>=0.22' || __zypper_install 'python-M2Crypto>=0.21' || return 1 + + if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" # shellcheck disable=SC2086 @@ -5829,6 +5830,11 @@ install_suse_11_stable_deps() { # shellcheck disable=SC2086,SC2090 __zypper_install ${__PACKAGES} || return 1 + # SLES 11 SP3 ships with both python-M2Crypto-0.22.* and python-m2crypto-0.21 and we will be asked which + # we want to install, even with --non-interactive. + # Let's try to install the higher version first and then the lower one in case of failure + __zypper_install 'python-M2Crypto>=0.22' || __zypper_install 'python-M2Crypto>=0.21' || return 1 + if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" # shellcheck disable=SC2086