diff --git a/.kitchen.yml b/.kitchen.yml index 102988a..886b4f1 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -46,6 +46,11 @@ platforms: run_command: /sbin/init provision_command: - yum install -y upstart + - name: amazon-2 + driver_config: + image: amazonlinux:2 + platform: rhel + run_command: /usr/lib/systemd/systemd - name: ubuntu-18.04 driver_config: run_command: /lib/systemd/systemd @@ -83,12 +88,14 @@ suites: - name: py2-git-2017.7 provisioner: salt_version: 2017.7 + excludes: + - arch - name: py2-git-2018.3 provisioner: salt_version: 2018.3 - - name: py2-git-fluorine + - name: py2-git-2019.2 provisioner: - salt_version: fluorine + salt_version: 2019.2 - name: py2-git-develop provisioner: salt_version: develop diff --git a/.travis.yml b/.travis.yml index 331b9a1..7232546 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,7 @@ env: - PLATFORM=centos-7 - PLATFORM=centos-6 - PLATFORM=amazon-1 + - PLATFORM=amazon-2 - PLATFORM=ubuntu-1804 - PLATFORM=ubuntu-1604 - PLATFORM=ubuntu-1404 diff --git a/AUTHORS.rst b/AUTHORS.rst index cd7c28f..c0a444a 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -38,11 +38,13 @@ Megan Wilhite Ch3LL megan.wilhite@gmail.com Chris Rebert cvrebert chris.rebert@hulu.com Chris Buechler cbuechler cmb@pfsense.org Christer Edwards cedwards +Christian McHugh mchugh19 Clark Perkins iclarkperkins clark.perkins@digitalreasoning.com Dag Viggo Lokøen dagvl dag.viggo@lokoen.org Dan Mick dmick dan.mick@inktank.com Daniel Poelzleithner poelzi Daniel Wallace gtmanfred danielwallace@gtmanfred.com +Darko Cerdic darkocerdic David J. Felix DavidJFelix denmat denmat Denys Havrysh vutny denys.gavrysh@gmail.com @@ -77,6 +79,7 @@ Jasper Lievisse Adriaanse jasperla jasper@humppa.nl JD decomposite Jeff Hui jeffh jeff@jeffhui.net Jeff Strunk jstrunk +Jorge Schrauwen sjorge sjorge@blackdot.be Juan A. Moyano wincus wincus.public@gmail.com Justin Anderson justinta justin.ta@outlook.com Justin Findlay jfindlay jfindlay@gmail.com @@ -94,6 +97,7 @@ Marcus Furlong furlongm furlongm@gmail.com Mark Lee malept markgaylard markgaylard Matt Black mafrosis +Matt McKinnon ripesensor Matthew Garrett cingeyedog matt@clemson.edu Matthew Mead-Briggs mattmb Matthew Richardson mrichar1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 11614af..ec421fd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -140,7 +140,9 @@ key to your GitHub account. Once these steps are completed, the commit signing verification will look like the example in GitHub's [GPG Signature Verification feature announcement](https://github.com/blog/2144-gpg-signature-verification). -## Release Cadence +## Release Information + +### Release Cadence There is no defined release schedule for the bootstrap script at this time. Typically, SaltStack's release team determines when it would be good to release @@ -154,6 +156,21 @@ Timing the release usually involves an analysis of the following: - Fixes needed for inclusion in an upcoming version of [Salt](https://github.com/saltstack/salt) - Length of time since the last bootstrap release +### Release Process + +The release process consists of the following steps: + +1. Merge in any outstanding PRs that are ready. +1. Add new contributors to the [AUTHORS](https://github.com/saltstack/salt-bootstrap/blob/develop/AUTHORS.rst) file. +1. Update the [ChangeLog](https://github.com/saltstack/salt-bootstrap/blob/develop/ChangeLog). +1. Update the version number in the bootstrap script. The version number is date-based, ``. + For example, version `2018.08.15` was released on August 15, 2018. +1. Merge the `develop` branch into the `stable` branch. +1. Update `bootstrap.saltstack.com` with the new stable release. The checksum on the + [README page](https://github.com/saltstack/salt-bootstrap/blob/develop/README.rst) + should also be updated. +1. Merge the new stable release into [Salt](https://github.com/saltstack/salt). + ## Adding Support for Other Operating Systems The following operating systems are detected, but Salt and its dependency diff --git a/ChangeLog b/ChangeLog index 551c497..b7b623b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,15 +1,23 @@ Version TBD (In Progress on the Develop Branch): +Version 2019.01.08: + * use official amazon linux 2 repo for amazon linux 2 (mchugh19) #1287 + * Add release info to applicable docs (rallytime) #1292 + * Modify wait_for_apt function (ripesensor) #1291 + * Add support for LinuxMint 19 (darkocerdic) #1289 + * FIX #1237 SmartOS should use pkgin show-deps (sjorge) #1283 + * Fedora: Reduce DNF calls to 2 during install_dep phase (The-Loeki) #1278 + Version 2018.08.15: - * Add tests using kitchen-salt #1279 - * Add python-futures to Py2 installs #1279 + * Add tests using kitchen-salt (gtmanfred) #1279 + * Add python-futures to Py2 installs (gtmanfred) #1279 Version 2018.08.13: * Fedora Py3 fixes (The-Loeki) #1273 * Handle commented lines in the requirements files for pip pkgs (rallytime) #1271 * Remove typo: extra 'c' was accidentally added in #1269 (rallytime) #1270 * [Arch] Add python2-futures to list of pkgs on git install (rallytime) #1269 - * Fix undefined variable warn_msg on amd64 (alexandruavadanii) #1268 + * Fix undefined variable warn_msg on amd64 (alexandruavadanii) #1268 * SLES12SP changed packages git to git-core and libzmq3 to libzmq4 (mfapouw) #1266 * Add opensuse 15 specific installation functions (rallytime) #1263 * Remove support for openSUSE Leap 42.2 (rallytime) #1262 @@ -353,10 +361,10 @@ Version 2015.05.04: * Add Debian 8 support. Thanks Matt Black(mafrosis) * Improve Debian version parsing. Thanks Mark Lee(malept) * Make sure we update packages list one Chris Lea's PPA repository is added. - * Hard code the Debian Squeeze backports to the DE mirror since the main repository is down. + * Hard code the Debian Squeeze backports to the DE mirror since the main repository is down. Thanks @panticz. #589. * Only install git if not already installed. #560 - * Fix openSUSE 13.2 where we need to pass --replaceflags. Thanks Roman Inflianskas(rominf). + * Fix openSUSE 13.2 where we need to pass --replaceflags. Thanks Roman Inflianskas(rominf). #504. * Make sure that a recent enough requests package is installed in Debian/Ubuntu. * Install tornado on git installs for the develop branch if necessary. #580 @@ -372,11 +380,11 @@ Version 2015.02.28: * Fix Debian backports repository. Version 2015.02.27: - * Try other tools besides wget when downloading the COPR repo file. Thanks Ronald van + * Try other tools besides wget when downloading the COPR repo file. Thanks Ronald van Zantvoort(The-Loeki) - * No need to install packages from the Unstable repository for debian, use backports. Thanks + * No need to install packages from the Unstable repository for debian, use backports. Thanks Ari Aosved(devaos) - * Fix an issue in CentOS where the syndic package wasn't being installed(since it's now a + * Fix an issue in CentOS where the syndic package wasn't being installed(since it's now a separate package). Thanks Ronald van Zantvoort(The-Loeki) * Enable the server-optionals repository for RHEL >= 7 * RHEL/CentOS 5 now uses the COPR repository. #533 @@ -424,33 +432,33 @@ Version 2014.09.09: * Complete `salt-api` services checking. #450 Version 2014.08.30: - * Skip service checks for `salt-api`, since this should be an opt-in service not necessarily + * Skip service checks for `salt-api`, since this should be an opt-in service not necessarily meant to start at boot time. * Distro Support Fixes: - * Also install the salt-api service on RHEL based distributions for git based + * Also install the salt-api service on RHEL based distributions for git based installations. * Properly detect Arch Linux when lsb-release is available * Updated the URL for EPEL 7 Version 2014.08.23: * Avoid redirect breakage when installing EPEL with rpm on RHEL 5 - * Ensure python-apt is installed by the bootstrap script for Debian & Ubuntu minions. Thanks + * Ensure python-apt is installed by the bootstrap script for Debian & Ubuntu minions. Thanks @garethgreenaway. * Don't shallow clone on git versions lower than 1.7.10 * Only shallow clone on git tag based installations * Configurable Salt repository clone directory for git based installations * Distro Support Fixed: * Fixed the URL to download EPEL for Cent 5 - * Use the full path to the `chkconfig` binary when checking for services in SysV init + * Use the full path to the `chkconfig` binary when checking for services in SysV init systems. - * Fixed an issue where the default sleep period(3 secs) on Ubuntu would cause a race - condition with upstart wherein the package installation would call an upstart start and - before it could complete, bootstrap would call another. The result was *two* copies of salt - running which ended up causing a most stubborn bug that's documented in + * Fixed an issue where the default sleep period(3 secs) on Ubuntu would cause a race + condition with upstart wherein the package installation would call an upstart start and + before it could complete, bootstrap would call another. The result was *two* copies of salt + running which ended up causing a most stubborn bug that's documented in https://github.com/saltstack/salt/issues/12248 Version 2014.07.29: - * Shallow clone Salt's repository for speed improvements. In case of failure, resume old + * Shallow clone Salt's repository for speed improvements. In case of failure, resume old behaviour. * Fixed bug introduced in 0577622 when salt-api service install and checks were added * Distro Support Fixed: @@ -459,7 +467,7 @@ Version 2014.07.29: Version 2014.07.27: * Amazon Linux AMI 2010.xx is not explicitly not supported. * Install the `salt-api` scripts if available when the `salt-master` is also installed. - * Added support for a configurable sleep time when starting, restarting and checking for + * Added support for a configurable sleep time when starting, restarting and checking for enabled services. * Drop the `tsflags` requirement for RHEL and RHEL based distributions. * When sorting release files, oracle-release has higher priority than redhat-release. @@ -470,10 +478,10 @@ Version 2014.07.27: * CentOS 7 now uses systemd and the script now properly handles it * systemd in openSUSE 12.2 complains if service does not contain `.service`` * Properly detect openSUSE using `lsb_release - * SLES 11 SP3 ships with both python-M2Crypto-0.22.* and python-m2crypto-0.21 and we will - be asked which we want to install, even with --non-interactive. Let's try to install the + * SLES 11 SP3 ships with both python-M2Crypto-0.22.* and python-m2crypto-0.21 and we will + be asked which we want to install, even with --non-interactive. Let's try to install the higher version first and then the lower one in case of failure. - * Allow some extra time on RHEL for the optionals repo check in case the repository + * Allow some extra time on RHEL for the optionals repo check in case the repository subscription is being managed externally. Version 2014.06.30: @@ -494,7 +502,7 @@ Version 2014.06.21: Version 2014.06.19: * Allow passing the master address as an environment variable, `BS_SALT_MASTER_ADDRESS` - * Fixed an issue with the keys pre-seed. We were passing absolute paths where we only needed + * Fixed an issue with the keys pre-seed. We were passing absolute paths where we only needed basenames. * Added HTTP proxy configuration support. Thanks Giuseppe Iannello(gianello), * Distro Support Added: @@ -507,14 +515,14 @@ Version 2014.06.19: Version 2014.04.16: * Fixed a bug for RHEL 6 based distributions where yum-utils was not getting installed. * Added minor version check for RHEL 6 optional channel. - * Added quotes around "apache-libcloud>=$_LIBCLOUD_MIN_VERSION" for proper version requirements + * Added quotes around "apache-libcloud>=$_LIBCLOUD_MIN_VERSION" for proper version requirements handling. * Install the python 'requests' package which is now a hard dependency in Salt. - * When installing from a user defined repository add the official one as a remote and fetch + * When installing from a user defined repository add the official one as a remote and fetch its tags for proper versioning. * Distro Support Fixed: * CentOS netinstall ISO's don't install `chkconfig` - * Improved RHEL optional repository detection. This allows user repository usage, which + * Improved RHEL optional repository detection. This allows user repository usage, which don't need the optional repository support since they usually provide their packages. * Distro Support Added: * Oracle Linux @@ -525,7 +533,7 @@ Version 2014.03.10-1: * Fix the Debian services running function Version 2014.03.10: - * Debian based distributions which don't use upstart now also check if the salt services are + * Debian based distributions which don't use upstart now also check if the salt services are enabled. * Distro Support Fixed: * RedHat based distributions now have a proper services enabled checker. @@ -544,25 +552,25 @@ Version 2014.02.19: * Fixed Fedora Git based installations(git was not being installed) Version 2014.02.18: - * Debian based distribution now get a warning stating that NOT starting daemons does not work + * Debian based distribution now get a warning stating that NOT starting daemons does not work as supposed, mainly because that's the Debian policy. - * Fix bug introduced when implementing the master ip flag. The default minion includes + * Fix bug introduced when implementing the master ip flag. The default minion includes directory is `minion.d`, not `minion.conf.d` Version 2014.02.16: - * The script now allows setting up the salt-master address as a separate configuration file by + * The script now allows setting up the salt-master address as a separate configuration file by passing `-A` to the script. - * Add support to install apache-libcloud by passing the `-L` flag. In some distribution it's - also needed to pass `-P` because the minimal apache-libcloud version is `0.14.0`. This support + * Add support to install apache-libcloud by passing the `-L` flag. In some distribution it's + also needed to pass `-P` because the minimal apache-libcloud version is `0.14.0`. This support is still missing for FreeBSD and SmartOS. - * Fixed an issue when copying or moving files. We now test to see if the destination is a - directory and create a full path from that so that the "do not override" logic works as + * Fixed an issue when copying or moving files. We now test to see if the destination is a + directory and create a full path from that so that the "do not override" logic works as supposed. #294. * Allow passing additional package names to install while installing Salt's dependencies. #262 - * Pass the salt configuration directory, default or from environment variable to the setup.py + * Pass the salt configuration directory, default or from environment variable to the setup.py script for git based installations. #305 * Distro Support Fixed: - * FreeBSD `fetch` now has a notion of insecure certificates. Handle this properly. Thank + * FreeBSD `fetch` now has a notion of insecure certificates. Handle this properly. Thank You Mike Carlson(m87carlson). * Arch, openSUSE and SuSE are now upgradable when the `-U` flag is passed. * Force overwrites now works for existing init.d scripts on CentOS git installations. #259 @@ -577,7 +585,7 @@ Version 1.5.10: * Salt no longer has the master branch in git, install from develop as default. * Installing from Git on Red Hat based distributions now also needs `yum-utils` installed. * Allow the script to use a different git repository to install from. - * Fixed a bug where a branch name with dashes would be wrongly detected as an option to the + * Fixed a bug where a branch name with dashes would be wrongly detected as an option to the script. * Default to secure file downloads(if any). * Distro Support Fixed: @@ -588,23 +596,23 @@ Version 1.5.9: * Allow to not start the daemons after bootstrapping salt. This will allow `vagrant-lxc` installations, `debootstrap*`, etc, to finish properly. Thanks Henrik Holmboe (holmboe). * Distro Support Fixed: - * Salt >= 0.17 requires ElementTree which is on the python standard library after python + * Salt >= 0.17 requires ElementTree which is on the python standard library after python 2.6 but openSUSE split that into a separate package. * Fixed a logic preventing proper Ubuntu bootstrapping on some situations. Version 1.5.8: - * Fixed an Ubuntu regression. `add-apt-repository` is only available on + * Fixed an Ubuntu regression. `add-apt-repository` is only available on `software-properties-common` after 12.10, inclusive. Thanks Diego Woitasen(diegows) Version 1.5.7: - * For RedHat based distributions which rely on `epel`, the user can now pass `testing` to the + * For RedHat based distributions which rely on `epel`, the user can now pass `testing` to the script and `epel-testing` shall be used to bootstrap salt and it's dependencies. - * No full system upgrades, if optional by the distribution, shall be done unless `-U` is passed - to the bootstrap script(required upgrade procedures must exist on the script, currently Debian + * No full system upgrades, if optional by the distribution, shall be done unless `-U` is passed + to the bootstrap script(required upgrade procedures must exist on the script, currently Debian and RedHat based distributions support system upgrades). * Fixed an issue where passing BS_KEEP_TEMP_FILES=1 to the script was causing an error. #206. - * Switched FreeBSD default packages repository to PCBSD(http://www.pcbsd.org) and added - multiple repository support to install salt from the SaltStack's FreeBSD repository. Thanks + * Switched FreeBSD default packages repository to PCBSD(http://www.pcbsd.org) and added + multiple repository support to install salt from the SaltStack's FreeBSD repository. Thanks Christer Edwards(cedwards). * Improved Gentoo Support. Thanks Elias Probst(eliasp). * Stop execution soon for end of life distributions or non supported distribution versions. @@ -612,7 +620,7 @@ Version 1.5.7: * Fixed an unbound variable while bootstraping Gentoo. * Fixed CentOS/RHEL 5. * Fixed crypto++ compilation. Thanks Kenneth Wilke(KennethWilke)! - * Fixed FreeBSD git installations not pointing to the proper salt configuration directory, + * Fixed FreeBSD git installations not pointing to the proper salt configuration directory, which on FreeBSD is '/usr/local/etc/salt'. * Fixed testing installation for Red Hat based distributions. Thanks Jeff Strunk(jstrunk) * Fixed wrong package name on Arch. Thanks Niels Abspoel(aboe76) @@ -620,23 +628,23 @@ Version 1.5.7: * Fixed SmartOS installation. Thanks Matthieu Guegan(mguegan). Version 1.5.6: - * If there's a `grains` file on the provided temporary configuration directory, move it to the + * If there's a `grains` file on the provided temporary configuration directory, move it to the proper place while moving the minion configuration file. - * Gentoo bootstraps can now use a bin host to provide binary packages, just set the + * Gentoo bootstraps can now use a bin host to provide binary packages, just set the `BS_GENTOO_USE_BINHOST` environment variable. - * If `BS_KEEP_TEMP_FILES=1` is found on the environment, the script will copy the files instead + * If `BS_KEEP_TEMP_FILES=1` is found on the environment, the script will copy the files instead of moving them. - * There were still some `mv` and `cp` occurrences which were not using their `{move,copy}file` - replacements which ended up on now respecting the "Do not override existing configuration" + * There were still some `mv` and `cp` occurrences which were not using their `{move,copy}file` + replacements which ended up on now respecting the "Do not override existing configuration" feature. * Distro Support Fixed: * Arch now upgrades it's system package properly as suggested on their mailing list. - * Arch now moves back any configuration files provided by the user renamed by pacman on the + * Arch now moves back any configuration files provided by the user renamed by pacman on the installation process. - * Fixed SmartOS detection(was being detected as Solaris) and bootstrapping. Fixed SmartOS + * Fixed SmartOS detection(was being detected as Solaris) and bootstrapping. Fixed SmartOS different gcc package names for different package sets. * Fixed FreeBSD git based installations(no rc.d scripts were available). - * Fixed FreeBSD not re-evaluating the `PKI_DIR` variable since the `SALT_ETC_DIR` was + * Fixed FreeBSD not re-evaluating the `PKI_DIR` variable since the `SALT_ETC_DIR` was redefined. * Distro Support Added: * Linux Mint. Thanks Alex Van't Hof(alexvh)! @@ -649,7 +657,7 @@ Version 1.5.5: * Debian installations now use SaltStack's repository. * Configuration files can now be passed as an URL to a compressed file. Thanks Geoff Garside! * Distro Support Fixed: - * Debian's optional ZMQ3 support was fixed (libzmq3 has moved from experimental to + * Debian's optional ZMQ3 support was fixed (libzmq3 has moved from experimental to unstable). * Ubuntu Lucid Daily PPA * SmartOS no longer ignores $SALT_ETC_DIR. Matthieu Guegan! @@ -664,10 +672,10 @@ Version 1.5.5: Version 1.5.4: * Fixed an issue we had when /proc/cpuinfo had more than one CPU. Detected on AMD CPUs. - * OpenSUSE 12.3 uses lsb_release. Fix the returned distro name "openSUSE project" to "openSUSE" + * OpenSUSE 12.3 uses lsb_release. Fix the returned distro name "openSUSE project" to "openSUSE" which the script handles. * Added an custom move function which will only override if required and if we permit it. - * Implemented the necessary function to pre-seed minion keys on a salt master as an optional + * Implemented the necessary function to pre-seed minion keys on a salt master as an optional argument. * Distro Support Fixed: * FreeBSD (Don't let the script fail if PACKAGESITE is not set) @@ -683,7 +691,7 @@ Version 1.5.3: * Fixed `config_salt()` * Distro Support Fixed: * EPEL-based installations (CentOS, Amazon Linux, RedHat) - * SuSE/OpenSUSE (problem running the script twice, ie, existing `devel_languages_python` + * SuSE/OpenSUSE (problem running the script twice, ie, existing `devel_languages_python` repository) * SuSE 11 SP1 (pip based install and config trigger) * Distro Support Added: @@ -705,7 +713,7 @@ Version 1.5.1: * Improved unittesting. * Starting daemons. * Make sure that daemons are really running. - * For the users to make the choice if installing from PIP (if required since there aren't system + * For the users to make the choice if installing from PIP (if required since there aren't system pacakges). * Fixed salt's git cloning when the salt git tree is already present on the system. * Distro Support Fixed: diff --git a/LICENSE b/LICENSE index 552a71d..31ccb14 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ Salt Bootstrap - Generic Salt Bootstrap Script - Copyright 2012-2017 Salt Stack (saltstack.org) + Copyright 2012-2019 SaltStack (saltstack.com) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,4 +13,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/README.rst b/README.rst index 1138886..65c1a03 100644 --- a/README.rst +++ b/README.rst @@ -28,6 +28,7 @@ sum** of the downloaded ``bootstrap-salt.sh`` file. The SHA256 sum of the ``bootstrap-salt.sh`` file, per release, is: +- 2018.08.15: ``6d414a39439a7335af1b78203f9d37e11c972b3c49c519742c6405e2944c6c4b`` - 2018.08.13: ``98284bdc2b5ebaeb619b22090374e42a68e8fdefe6bff1e73bd1760db4407ed0`` - 2018.04.25: ``e2e3397d6642ba6462174b4723f1b30d04229b75efc099a553e15ea727877dfb`` - 2017.12.13: ``c127b3aa4a8422f6b81f5b4a40d31d13cec97bf3a39bca9c11a28f24910a6895`` @@ -43,6 +44,9 @@ Contributing The Salt Bootstrap project is open and encouraging to code contributions. Please review the `Contributing Guidelines`_ for information on filing issues, fixing bugs, and submitting features. +The `Contributing Guidelines`_ also contain information about the Bootstrap release cadence and +process. + Examples -------- @@ -321,7 +325,7 @@ UNIX systems **SunOS**: -- SmartOS +- SmartOS (2015Q4 and later) Unsupported Distributions ------------------------- diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index c3af792..d52abe2 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -1,4 +1,9 @@ #!/bin/sh - + +# WARNING: Changes to this file in the salt repo will be overwritten! +# Please submit pull requests against the salt-bootstrap repo: +# https://github.com/saltstack/salt-bootstrap + #====================================================================================================================== # vim: softtabstop=4 shiftwidth=4 expandtab fenc=utf-8 spell spelllang=en cc=120 #====================================================================================================================== @@ -18,7 +23,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2018.08.15" +__ScriptVersion="2019.01.08" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -585,14 +590,14 @@ elif [ "$ITYPE" = "stable" ]; then if [ "$#" -eq 0 ];then STABLE_REV="latest" else - if [ "$(echo "$1" | grep -E '^(latest|1\.6|1\.7|2014\.1|2014\.7|2015\.5|2015\.8|2016\.3|2016\.11|2017\.7|2018\.3)$')" != "" ]; then + if [ "$(echo "$1" | grep -E '^(latest|1\.6|1\.7|2014\.1|2014\.7|2015\.5|2015\.8|2016\.3|2016\.11|2017\.7|2018\.3|2019\.2)$')" != "" ]; then STABLE_REV="$1" shift elif [ "$(echo "$1" | grep -E '^([0-9]*\.[0-9]*\.[0-9]*)$')" != "" ]; then STABLE_REV="archive/$1" shift else - echo "Unknown stable version: $1 (valid: 1.6, 1.7, 2014.1, 2014.7, 2015.5, 2015.8, 2016.3, 2016.11, 2017.7, 2018.3, latest, \$MAJOR.\$MINOR.\$PATCH)" + echo "Unknown stable version: $1 (valid: 1.6, 1.7, 2014.1, 2014.7, 2015.5, 2015.8, 2016.3, 2016.11, 2017.7, 2018.3, 2019.2, latest, \$MAJOR.\$MINOR.\$PATCH)" exit 1 fi fi @@ -1269,6 +1274,7 @@ __ubuntu_derivatives_translation() { linuxmint_13_ubuntu_base="12.04" linuxmint_17_ubuntu_base="14.04" linuxmint_18_ubuntu_base="16.04" + linuxmint_19_ubuntu_base="18.04" linaro_12_ubuntu_base="12.04" elementary_os_02_ubuntu_base="12.04" neon_16_ubuntu_base="16.04" @@ -1632,7 +1638,8 @@ __check_end_of_life_versions() { amazon*linux*ami) # Amazon Linux versions lower than 2012.0X no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 2012 ]; then + # Except for Amazon Linux 2, which reset the major version counter + if [ "$DISTRO_MAJOR_VERSION" -lt 2012 ] && [ "$DISTRO_MAJOR_VERSION" -gt 10 ]; then echoerror "End of life distributions are not supported." echoerror "Please consider upgrading to the next stable. See:" echoerror " https://aws.amazon.com/amazon-linux-ami/" @@ -1797,24 +1804,32 @@ __function_defined() { # process is finished so the script doesn't exit on a locked proc. #---------------------------------------------------------------------------------------------------------------------- __wait_for_apt(){ - echodebug "Checking if apt process is currently running." - # Timeout set at 15 minutes WAIT_TIMEOUT=900 - while ps -C apt,apt-get,aptitude,dpkg >/dev/null; do - sleep 1 - WAIT_TIMEOUT=$((WAIT_TIMEOUT - 1)) + # Run our passed in apt command + "${@}" + APT_RETURN=$? - # If timeout reaches 0, abort. - if [ "$WAIT_TIMEOUT" -eq 0 ]; then - echoerror "Apt, apt-get, aptitude, or dpkg process is taking too long." - echoerror "Bootstrap script cannot proceed. Aborting." - return 1 - fi + # If our exit code from apt is 100, then we're waiting on a lock + while [ $APT_RETURN -eq 100 ]; do + echoinfo "Aware of the lock. Patiently waiting $WAIT_TIMEOUT more seconds..." + sleep 1 + WAIT_TIMEOUT=$((WAIT_TIMEOUT - 1)) + + # If timeout reaches 0, abort. + if [ "$WAIT_TIMEOUT" -eq 0 ]; then + echoerror "Apt, apt-get, aptitude, or dpkg process is taking too long." + echoerror "Bootstrap script cannot proceed. Aborting." + return 1 + else + # Try running apt again until our return code != 100 + "${@}" + APT_RETURN=$? + fi done - echodebug "No apt processes are currently running." + return $APT_RETURN } #--- FUNCTION ------------------------------------------------------------------------------------------------------- @@ -1823,8 +1838,7 @@ __wait_for_apt(){ # PARAMETERS: packages #---------------------------------------------------------------------------------------------------------------------- __apt_get_install_noinput() { - __wait_for_apt - apt-get install -y -o DPkg::Options::=--force-confold "${@}"; return $? + __wait_for_apt apt-get install -y -o DPkg::Options::=--force-confold "${@}"; return $? } # ---------- end of function __apt_get_install_noinput ---------- @@ -1833,8 +1847,7 @@ __apt_get_install_noinput() { # DESCRIPTION: (DRY) apt-get upgrade with noinput options #---------------------------------------------------------------------------------------------------------------------- __apt_get_upgrade_noinput() { - __wait_for_apt - apt-get upgrade -y -o DPkg::Options::=--force-confold; return $? + __wait_for_apt apt-get upgrade -y -o DPkg::Options::=--force-confold; return $? } # ---------- end of function __apt_get_upgrade_noinput ---------- @@ -1844,11 +1857,10 @@ __apt_get_upgrade_noinput() { # PARAMETERS: url #---------------------------------------------------------------------------------------------------------------------- __apt_key_fetch() { - __wait_for_apt url=$1 # shellcheck disable=SC2086 - apt-key adv ${_GPG_ARGS} --fetch-keys "$url"; return $? + __wait_for_apt apt-key adv ${_GPG_ARGS} --fetch-keys "$url"; return $? } # ---------- end of function __apt_key_fetch ---------- @@ -2633,8 +2645,7 @@ __install_saltstack_ubuntu_repository() { __apt_key_fetch "$SALTSTACK_UBUNTU_URL/SALTSTACK-GPG-KEY.pub" || return 1 - __wait_for_apt - apt-get update || return 1 + __wait_for_apt apt-get update || return 1 } install_ubuntu_deps() { @@ -2646,8 +2657,7 @@ install_ubuntu_deps() { __enable_universe_repository || return 1 - __wait_for_apt - apt-get update || return 1 + __wait_for_apt apt-get update || return 1 fi __PACKAGES='' @@ -2703,8 +2713,7 @@ install_ubuntu_stable_deps() { # No user interaction, libc6 restart services for example export DEBIAN_FRONTEND=noninteractive - __wait_for_apt - apt-get update || return 1 + __wait_for_apt apt-get update || return 1 if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then if [ "${_INSECURE_DL}" -eq $BS_TRUE ]; then @@ -2724,8 +2733,7 @@ install_ubuntu_stable_deps() { } install_ubuntu_git_deps() { - __wait_for_apt - apt-get update || return 1 + __wait_for_apt apt-get update || return 1 if ! __check_command_exists git; then __apt_get_install_noinput git-core || return 1 @@ -3032,8 +3040,7 @@ __install_saltstack_debian_repository() { __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALTSTACK-GPG-KEY.pub" || return 1 - __wait_for_apt - apt-get update || return 1 + __wait_for_apt apt-get update || return 1 } install_debian_deps() { @@ -3044,8 +3051,7 @@ install_debian_deps() { # No user interaction, libc6 restart services for example export DEBIAN_FRONTEND=noninteractive - __wait_for_apt - apt-get update || return 1 + __wait_for_apt apt-get update || return 1 if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then # Try to update GPG keys first if allowed @@ -3164,8 +3170,7 @@ install_debian_8_git_deps() { /etc/apt/sources.list.d/backports.list fi - __wait_for_apt - apt-get update || return 1 + __wait_for_apt apt-get update || return 1 # python-tornado package should be installed from backports repo __PACKAGES="${__PACKAGES} python-backports.ssl-match-hostname python-tornado/jessie-backports" @@ -3415,36 +3420,33 @@ install_debian_check_services() { # install_fedora_deps() { + if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then + dnf -y update || return 1 + fi + + __PACKAGES="${__PACKAGES:=}" if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then # Packages are named python3- PY_PKG_VER=3 - __PACKAGES="python3-m2crypto python3-PyYAML" + __PACKAGES="${__PACKAGES} python3-m2crypto python3-PyYAML" else PY_PKG_VER=2 - __PACKAGES="m2crypto" + __PACKAGES="${__PACKAGES} m2crypto" if [ "$DISTRO_MAJOR_VERSION" -ge 28 ]; then __PACKAGES="${__PACKAGES} python2-pyyaml" else __PACKAGES="${__PACKAGES} PyYAML" fi fi - - __PACKAGES="${__PACKAGES} procps-ng dnf-utils libyaml python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2" + __PACKAGES="${__PACKAGES} dnf-utils libyaml procps-ng python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2" __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests python${PY_PKG_VER}-zmq" - - # shellcheck disable=SC2086 - dnf install -y ${__PACKAGES} || return 1 - - if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then - dnf -y update || return 1 - fi - if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" - # shellcheck disable=SC2086 - dnf install -y ${_EXTRA_PACKAGES} || return 1 fi + # shellcheck disable=SC2086 + dnf install -y ${__PACKAGES} ${_EXTRA_PACKAGES} || return 1 + return 0 } @@ -3494,36 +3496,38 @@ install_fedora_git_deps() { PY_PKG_VER=2 fi + __PACKAGES="${__PACKAGES:=}" if [ "$_INSECURE_DL" -eq $BS_FALSE ] && [ "${_SALT_REPO_URL%%://*}" = "https" ]; then - dnf install -y ca-certificates || return 1 + __PACKAGES="${__PACKAGES} ca-certificates" + fi + if ! __check_command_exists git; then + __PACKAGES="${__PACKAGES} git" + fi + if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-libcloud python${PY_PKG_VER}-netaddr" + fi + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd" + + # Fedora 28+ ships with tornado 5.0+ which is broken for salt on py3 + # https://github.com/saltstack/salt-bootstrap/issues/1220 + if [ "${PY_PKG_VER}" -lt 3 ] || [ "$DISTRO_MAJOR_VERSION" -lt 28 ]; then + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-tornado" fi install_fedora_deps || return 1 - if ! __check_command_exists git; then - dnf install -y git || return 1 - fi - __git_clone_and_checkout || return 1 - __PACKAGES="python${PY_PKG_VER}-systemd" + # Fedora 28+ needs tornado <5.0 from pip + # https://github.com/saltstack/salt-bootstrap/issues/1220 if [ "${PY_PKG_VER}" -eq 3 ] && [ "$DISTRO_MAJOR_VERSION" -ge 28 ]; then __check_pip_allowed "You need to allow pip based installations (-P) for Tornado <5.0 in order to install Salt on Python 3" grep tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" | while IFS=' ' read -r dep; do "${_PY_EXE}" -m pip install "${dep}" || return 1 done - else - __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-tornado" fi - if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-libcloud python${PY_PKG_VER}-netaddr" - fi - - # shellcheck disable=SC2086 - dnf install -y ${__PACKAGES} || return 1 - # Let's trigger config_salt() if [ "$_TEMP_CONFIG_DIR" = "null" ]; then _TEMP_CONFIG_DIR="${_SALT_GIT_CHECKOUT_DIR}/conf/" @@ -4681,6 +4685,138 @@ install_amazon_linux_ami_git_deps() { return 0 } +install_amazon_linux_ami_2_git_deps() { + if [ "$_INSECURE_DL" -eq $BS_FALSE ] && [ "${_SALT_REPO_URL%%://*}" = "https" ]; then + yum -y install ca-certificates || return 1 + fi + + PIP_EXE='pip' + if __check_command_exists python2.7; then + if ! __check_command_exists pip2.7; then + __yum_install_noinput python2-pip + fi + PIP_EXE='/bin/pip' + _PY_EXE='python2.7' + fi + + install_amazon_linux_ami_2_deps || return 1 + + if ! __check_command_exists git; then + __yum_install_noinput git || return 1 + fi + + __git_clone_and_checkout || return 1 + + __PACKAGES="" + __PIP_PACKAGES="" + + if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then + __check_pip_allowed "You need to allow pip based installations (-P) in order to install apache-libcloud" + __PACKAGES="${__PACKAGES} python27-pip" + __PIP_PACKAGES="${__PIP_PACKAGES} apache-libcloud>=$_LIBCLOUD_MIN_VERSION" + fi + + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" ]; then + # We're on the develop branch, install whichever tornado is on the requirements file + __REQUIRED_TORNADO="$(grep tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt")" + if [ "${__REQUIRED_TORNADO}" != "" ]; then + __PACKAGES="${__PACKAGES} ${pkg_append}-tornado" + fi + fi + + if [ "${__PACKAGES}" != "" ]; then + # shellcheck disable=SC2086 + __yum_install_noinput ${__PACKAGES} || return 1 + fi + + if [ "${__PIP_PACKAGES}" != "" ]; then + # shellcheck disable=SC2086 + ${PIP_EXE} install ${__PIP_PACKAGES} || return 1 + fi + + # Let's trigger config_salt() + if [ "$_TEMP_CONFIG_DIR" = "null" ]; then + _TEMP_CONFIG_DIR="${_SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_amazon_linux_ami_2_deps() { + # Shim to figure out if we're using old (rhel) or new (aws) rpms. + _USEAWS=$BS_FALSE + pkg_append="python" + + if [ "$ITYPE" = "stable" ]; then + repo_rev="$STABLE_REV" + else + repo_rev="latest" + fi + + if echo $repo_rev | grep -E -q '^archive'; then + year=$(echo "$repo_rev" | cut -d '/' -f 2 | cut -c1-4) + else + year=$(echo "$repo_rev" | cut -c1-4) + fi + + if echo "$repo_rev" | grep -E -q '^(latest|2016\.11)$' || \ + [ "$year" -gt 2016 ]; then + _USEAWS=$BS_TRUE + pkg_append="python" + fi + + # We need to install yum-utils before doing anything else when installing on + # Amazon Linux ECS-optimized images. See issue #974. + __yum_install_noinput yum-utils + + # Do upgrade early + if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then + yum -y update || return 1 + fi + + if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then + __REPO_FILENAME="saltstack-repo.repo" + + base_url="$HTTP_VAL://${_REPO_URL}/yum/redhat/7/\$basearch/$repo_rev/" + base_url="$HTTP_VAL://${_REPO_URL}/yum/amazon/2/\$basearch/latest/" + gpg_key="${base_url}SALTSTACK-GPG-KEY.pub + ${base_url}base/RPM-GPG-KEY-CentOS-7" + repo_name="SaltStack repo for Amazon Linux 2.0" + + # This should prob be refactored to use __install_saltstack_rhel_repository() + # With args passed in to do the right thing. Reformatted to be more like the + # amazon linux yum file. + if [ ! -s "/etc/yum.repos.d/${__REPO_FILENAME}" ]; then + cat <<_eof > "/etc/yum.repos.d/${__REPO_FILENAME}" +[saltstack-repo] +name=$repo_name +failovermethod=priority +priority=10 +gpgcheck=1 +gpgkey=$gpg_key +baseurl=$base_url +_eof + fi + + fi + + # Package python-ordereddict-1.1-2.el6.noarch is obsoleted by python26-2.6.9-2.88.amzn1.x86_64 + # which is already installed + __PACKAGES="m2crypto ${pkg_append}-crypto ${pkg_append}-jinja2 PyYAML" + __PACKAGES="${__PACKAGES} ${pkg_append}-msgpack ${pkg_append}-requests ${pkg_append}-zmq" + __PACKAGES="${__PACKAGES} ${pkg_append}-futures" + + # shellcheck disable=SC2086 + __yum_install_noinput ${__PACKAGES} || return 1 + + if [ "${_EXTRA_PACKAGES}" != "" ]; then + echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" + # shellcheck disable=SC2086 + __yum_install_noinput ${_EXTRA_PACKAGES} || return 1 + fi +} + install_amazon_linux_ami_stable() { install_centos_stable || return 1 return 0 @@ -4715,6 +4851,41 @@ install_amazon_linux_ami_testing_post() { install_centos_testing_post || return 1 return 0 } + +install_amazon_linux_ami_2_stable() { + install_centos_stable || return 1 + return 0 +} + +install_amazon_linux_ami_2_stable_post() { + install_centos_stable_post || return 1 + return 0 +} + +install_amazon_linux_ami_2_restart_daemons() { + install_centos_restart_daemons || return 1 + return 0 +} + +install_amazon_linux_ami_2_git() { + install_centos_git || return 1 + return 0 +} + +install_amazon_linux_ami_2_git_post() { + install_centos_git_post || return 1 + return 0 +} + +install_amazon_linux_ami_2_testing() { + install_centos_testing || return 1 + return 0 +} + +install_amazon_linux_ami_2_testing_post() { + install_centos_testing_post || return 1 + return 0 +} # # Ended Amazon Linux AMI Install Functions # @@ -5336,7 +5507,8 @@ install_openbsd_restart_daemons() { # SmartOS Install Functions # install_smartos_deps() { - pkgin -y install zeromq py27-crypto py27-m2crypto py27-msgpack py27-yaml py27-jinja2 py27-zmq py27-requests || return 1 + smartos_deps="$(pkgin show-deps salt | grep '^\s' | grep -v '\snot' | xargs) py27-m2crypto" + pkgin -y install "${smartos_deps}" || return 1 # Set _SALT_ETC_DIR to SmartOS default if they didn't specify _SALT_ETC_DIR=${BS_SALT_ETC_DIR:-/opt/local/etc/salt}