diff --git a/.gitignore b/.gitignore index d8fe4fa..f7834ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /.project +*.swp diff --git a/AUTHORS.rst b/AUTHORS.rst index e8e82c5..bee1cfd 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -25,6 +25,7 @@ Jeff Hui jeffh jeff@jeffhui.net Jeff Strunk jstrunk Karl Grzeszczak karlgrz Kenneth Wilke KennethWilke +Liu Xiaohui oreh herolxh@gmail.com Matthew Willson ixela Matthieu Guegan mguegan Mike Carlson m87carlson mike@bayphoto.com diff --git a/ChangeLog b/ChangeLog index c12d6c5..7b4a92f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,17 @@ +Version 2014.04:16 + * Fixed a bug for RHEL 6 based distributions where yum-utils was not getting installed. + * Added minor version check for RHEL 6 optional channel. + * Added quotes around "apache-libcloud>=$_LIBCLOUD_MIN_VERSION" for proper version requirements + handling. + * Install the python 'requests' package which is now a hard dependency in Salt. + * When installing from a user defined repository add the official one as a remote and fetch + its tags for proper versioning. + * Distro Support Fixed: + * CentOS netinstall ISO's don't install `chkconfig` + * Distro Support Added: + * Oracle Linux + * Scientific Linux + Version 2014.03.10-1: * Distro Support Fixed: * Fix the Debian services running function diff --git a/README.rst b/README.rst index 9694efc..6ee429a 100644 --- a/README.rst +++ b/README.rst @@ -34,7 +34,7 @@ The URL used is just an HTTP redirect and as such it **will**, most likely, make ``wget`` or ``fetch`` (in FreeBSD >= 10), to complain about certificate issues. If this worries you, you **should not** use this URL. Use instead:: - https://github.com/saltstack/salt-bootstrap/raw/develop/bootstrap-salt.sh + https://github.com/saltstack/salt-bootstrap/raw/stable/bootstrap-salt.sh Examples @@ -362,8 +362,10 @@ Supported Operating Systems - Linaro - Linux Mint 13/14 - OpenSUSE 12.x +- Oracle Linux 5/5 - Red Hat 5/6 - Red Hat Enterprise 5/6 +- Scientific Linux 5/6 - SmartOS - SuSE 11 SP1/11 SP2 - Ubuntu 10.x/11.x/12.x/13.04/13.10 @@ -391,4 +393,28 @@ If after trying this, you still see the same problems, then, please `file an iss .. _`Salt`: http://saltstack.org/ .. _`file an issue`: https://github.com/saltstack/salt-bootstrap/issues/new + +Unsupported Distro +------------------ + +You found a Linux distribution which we still do not support or we do not correctly identify? +Please run the following commands and report their output when creating a ticket: + +.. code:: console + + sudo find /etc/ -name '*-release' -print -exec cat {} \; + which lsb_release && lsb_release -a + + +Testing in Vagrant +------------------ +You can use Vagrant_ to easily test changes on a clean machine. The ``Vagrantfile`` defaults to an +Ubuntu box. First, install Vagrant, then:: + + $ vagrant up + $ vagrant ssh + $ cd /salt_bootstrap + $ sudo sh salt-bootstrap.sh + +.. _Vagrant: http://www.vagrantup.com .. vim: fenc=utf-8 spell spl=en cc=100 tw=99 fo=want sts=2 sw=2 et diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..94166f9 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,118 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + # All Vagrant configuration is done here. The most common configuration + # options are documented and commented below. For a complete reference, + # please see the online documentation at vagrantup.com. + + # Every Vagrant virtual environment requires a box to build off of. + config.vm.box = "precise64" + + # The url from where the 'config.vm.box' box will be fetched if it + # doesn't already exist on the user's system. + # config.vm.box_url = "http://domain.com/path/to/above.box" + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # config.vm.network :forwarded_port, guest: 80, host: 8080 + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + # config.vm.network :private_network, ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network :public_network + + # If true, then any SSH connections made will enable agent forwarding. + # Default value: false + # config.ssh.forward_agent = true + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + config.vm.synced_folder ".", "/salt_bootstrap" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + # config.vm.provider :virtualbox do |vb| + # # Don't boot with headless mode + # vb.gui = true + # + # # Use VBoxManage to customize the VM. For example to change memory: + # vb.customize ["modifyvm", :id, "--memory", "1024"] + # end + # + # View the documentation for the provider you're using for more + # information on available options. + + # Enable provisioning with Puppet stand alone. Puppet manifests + # are contained in a directory path relative to this Vagrantfile. + # You will need to create the manifests directory and a manifest in + # the file precise64.pp in the manifests_path directory. + # + # An example Puppet manifest to provision the message of the day: + # + # # group { "puppet": + # # ensure => "present", + # # } + # # + # # File { owner => 0, group => 0, mode => 0644 } + # # + # # file { '/etc/motd': + # # content => "Welcome to your Vagrant-built virtual machine! + # # Managed by Puppet.\n" + # # } + # + # config.vm.provision :puppet do |puppet| + # puppet.manifests_path = "manifests" + # puppet.manifest_file = "site.pp" + # end + + # Enable provisioning with chef solo, specifying a cookbooks path, roles + # path, and data_bags path (all relative to this Vagrantfile), and adding + # some recipes and/or roles. + # + # config.vm.provision :chef_solo do |chef| + # chef.cookbooks_path = "../my-recipes/cookbooks" + # chef.roles_path = "../my-recipes/roles" + # chef.data_bags_path = "../my-recipes/data_bags" + # chef.add_recipe "mysql" + # chef.add_role "web" + # + # # You may also specify custom JSON attributes: + # chef.json = { :mysql_password => "foo" } + # end + + # Enable provisioning with chef server, specifying the chef server URL, + # and the path to the validation key (relative to this Vagrantfile). + # + # The Opscode Platform uses HTTPS. Substitute your organization for + # ORGNAME in the URL and validation key. + # + # If you have your own Chef Server, use the appropriate URL, which may be + # HTTP instead of HTTPS depending on your configuration. Also change the + # validation key to validation.pem. + # + # config.vm.provision :chef_client do |chef| + # chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME" + # chef.validation_key_path = "ORGNAME-validator.pem" + # end + # + # If you're using the Opscode platform, your validator client is + # ORGNAME-validator, replacing ORGNAME with your organization name. + # + # If you have your own Chef Server, the default validation client name is + # chef-validator, unless you changed the configuration. + # + # chef.validation_client_name = "ORGNAME-validator" +end diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 21305c3..cae269d 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -17,7 +17,7 @@ # CREATED: 10/15/2012 09:49:37 PM WEST #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2014.03.10-1" +__ScriptVersion="2014.04.16" __ScriptName="bootstrap-salt.sh" #====================================================================================================================== @@ -71,7 +71,7 @@ __detect_color_support # DESCRIPTION: Echo errors to stderr. #---------------------------------------------------------------------------------------------------------------------- echoerror() { - printf "${RC} * ERROR${EC}: $@\n" 1>&2; + printf "${RC} * ERROR${EC}: %s\n" "$@" 1>&2; } #--- FUNCTION ------------------------------------------------------------------------------------------------------- @@ -184,13 +184,13 @@ usage() { - git Examples: - $ ${__ScriptName} - $ ${__ScriptName} stable - $ ${__ScriptName} daily - $ ${__ScriptName} git - $ ${__ScriptName} git develop - $ ${__ScriptName} git v0.17.0 - $ ${__ScriptName} git 8c3fadf15ec183e5ce8c63739850d543617e4357 + - ${__ScriptName} + - ${__ScriptName} stable + - ${__ScriptName} daily + - ${__ScriptName} git + - ${__ScriptName} git develop + - ${__ScriptName} git v0.17.0 + - ${__ScriptName} git 8c3fadf15ec183e5ce8c63739850d543617e4357 Options: -h Display this message @@ -220,6 +220,8 @@ usage() { example, pass '--no-check-certificate' to 'wget' or '--insecure' to 'curl' -A Pass the salt-master DNS name or IP. This will be stored under \${BS_SALT_ETC_DIR}/minion.d/99-master-address.conf + -i Pass the salt-minion id. This will be stored under + \${BS_SALT_ETC_DIR}/minion_id -L Install the Apache Libcloud package if possible(required for salt-cloud) -p Extra-package to install while installing salt dependencies. One package per -p flag. You're responsible for providing the proper package name. @@ -235,6 +237,7 @@ EOT _KEEP_TEMP_FILES=${BS_KEEP_TEMP_FILES:-$BS_FALSE} _TEMP_CONFIG_DIR="null" _SALTSTACK_REPO_URL="git://github.com/saltstack/salt.git" +_SALT_REPO_URL=${_SALTSTACK_REPO_URL} _TEMP_KEYS_DIR="null" _INSTALL_MASTER=$BS_FALSE _INSTALL_SYNDIC=$BS_FALSE @@ -255,12 +258,13 @@ _WGET_ARGS=${BS_WGET_ARGS:-} _CURL_ARGS=${BS_CURL_ARGS:-} _FETCH_ARGS=${BS_FETCH_ARGS:-} _SALT_MASTER_ADDRESS="null" +_SALT_MINION_ID="null" # __SIMPLIFY_VERSION is mostly used in Solaris based distributions __SIMPLIFY_VERSION=$BS_TRUE _LIBCLOUD_MIN_VERSION="0.14.0" _EXTRA_PACKAGES="" -while getopts ":hvnDc:g:k:MSNXCPFUKIA:Lp:" opt +while getopts ":hvnDc:g:k:MSNXCPFUKIA:i:Lp:" opt do case "${opt}" in @@ -280,7 +284,7 @@ do exit 1 fi ;; - g ) _SALTSTACK_REPO_URL=$OPTARG ;; + g ) _SALT_REPO_URL=$OPTARG ;; k ) _TEMP_KEYS_DIR="$OPTARG" # If the configuration directory does not exist, error out if [ ! -d "$_TEMP_KEYS_DIR" ]; then @@ -299,6 +303,7 @@ do K ) _KEEP_TEMP_FILES=$BS_TRUE ;; I ) _INSECURE_DL=$BS_TRUE ;; A ) _SALT_MASTER_ADDRESS=$OPTARG ;; + i ) _SALT_MINION_ID=$OPTARG ;; L ) _INSTALL_CLOUD=$BS_TRUE ;; p ) _EXTRA_PACKAGES="$_EXTRA_PACKAGES $OPTARG" ;; @@ -311,7 +316,7 @@ do esac # --- end of case --- done -shift $(($OPTIND-1)) +shift $((OPTIND-1)) __check_unparsed_options() { @@ -346,7 +351,13 @@ fi # Check that we're installing a minion if we're being passed a master address if [ $_INSTALL_MINION -eq $BS_FALSE ] && [ $_SALT_MASTER_ADDRESS != "null" ]; then - echoerror "Don't pass a master address(-A) if no minion is going to be bootstrapped." + echoerror "Don't pass a master address (-A) if no minion is going to be bootstrapped." + exit 1 +fi + +# Check that we're installing a minion if we're being passed a master address +if [ $_INSTALL_MINION -eq $BS_FALSE ] && [ $_SALT_MINION_ID != "null" ]; then + echoerror "Don't pass a minion id (-i) if no minion is going to be bootstrapped." exit 1 fi @@ -400,7 +411,7 @@ if [ "$(${whoami})" != "root" ]; then fi # Let's discover how we're being called -CALLER=$(echo `ps -a -o pid,args | grep $$ | grep -v grep | tr -s ' '` | cut -d ' ' -f 2) +CALLER="$(echo `ps -a -o pid,args | grep $$ | grep -v grep | tr -s ' '` | cut -d ' ' -f 2)" if [ "${CALLER}x" = "${0}x" ]; then CALLER="PIPED THROUGH" fi @@ -571,7 +582,7 @@ __parse_version_string() { # DESCRIPTION: Strip single or double quotes from the provided string. #---------------------------------------------------------------------------------------------------------------------- __unquote_string() { - echo $@ | sed "s/^\([\"']\)\(.*\)\1\$/\2/g" + echo "${@}" | sed "s/^\([\"']\)\(.*\)\1\$/\2/g" } #--- FUNCTION ------------------------------------------------------------------------------------------------------- @@ -579,7 +590,7 @@ __unquote_string() { # DESCRIPTION: Convert CamelCased strings to Camel_Cased #---------------------------------------------------------------------------------------------------------------------- __camelcase_split() { - echo $@ | sed -r 's/([^A-Z-])([A-Z])/\1 \2/g' + echo "${@}" | sed -r 's/([^A-Z-])([A-Z])/\1 \2/g' } #--- FUNCTION ------------------------------------------------------------------------------------------------------- @@ -587,7 +598,7 @@ __camelcase_split() { # DESCRIPTION: Strip duplicate strings #---------------------------------------------------------------------------------------------------------------------- __strip_duplicates() { - echo $@ | tr -s '[:space:]' '\n' | awk '!x[$0]++' + echo "${@}" | tr -s '[:space:]' '\n' | awk '!x[$0]++' } #--- FUNCTION ------------------------------------------------------------------------------------------------------- @@ -598,11 +609,11 @@ __strip_duplicates() { __sort_release_files() { KNOWN_RELEASE_FILES=$(echo "(arch|centos|debian|ubuntu|fedora|redhat|suse|\ mandrake|mandriva|gentoo|slackware|turbolinux|unitedlinux|lsb|system|\ - os)(-|_)(release|version)" | sed -r 's:[[:space:]]::g') + oracle|os)(-|_)(release|version)" | sed -r 's:[[:space:]]::g') primary_release_files="" secondary_release_files="" # Sort know VS un-known files first - for release_file in $(echo $@ | sed -r 's:[[:space:]]:\n:g' | sort --unique --ignore-case); do + for release_file in $(echo "${@}" | sed -r 's:[[:space:]]:\n:g' | sort --unique --ignore-case); do match=$(echo $release_file | egrep -i ${KNOWN_RELEASE_FILES}) if [ "x${match}" != "x" ]; then primary_release_files="${primary_release_files} ${release_file}" @@ -643,17 +654,20 @@ __gather_linux_system_info() { rv=$(lsb_release >/dev/null 2>&1) if [ $? -eq 0 ]; then DISTRO_NAME=$(lsb_release -si) - if [ "x$(echo "$DISTRO_NAME" | grep RedHat)" != "x" ]; then + if [ "${DISTRO_NAME}" = "Scientific" ]; then + DISTRO_NAME="Scientific Linux" + elif [ "x$(echo "$DISTRO_NAME" | grep RedHat)" != "x" ]; then # Let's convert CamelCase to Camel Case DISTRO_NAME=$(__camelcase_split "$DISTRO_NAME") - fi - if [ "${DISTRO_NAME}" = "openSUSE project" ]; then + elif [ "${DISTRO_NAME}" = "openSUSE project" ]; then # lsb_release -si returns "openSUSE project" on openSUSE 12.3 DISTRO_NAME="opensuse" - fi - if [ "${DISTRO_NAME}" = "SUSE LINUX" ]; then + elif [ "${DISTRO_NAME}" = "SUSE LINUX" ]; then # lsb_release -si returns "SUSE LINUX" on SLES 11 SP3 DISTRO_NAME="suse" + elif [ "${DISTRO_NAME}" = "EnterpriseEnterpriseServer" ]; then + # This the Oracle Linux Enterprise ID before ORACLE LINUX 5 UPDATE 3 + DISTRO_NAME="Oracle Linux" fi rv=$(lsb_release -sr) [ "${rv}x" != "x" ] && DISTRO_VERSION=$(__parse_version_string "$rv") @@ -686,6 +700,8 @@ __gather_linux_system_info() { redhat ) if [ ".$(egrep 'CentOS' /etc/${rsource})" != . ]; then n="CentOS" + elif [ ".$(egrep 'Scientific' /etc/${rsource})" != . ]; then + n="Scientific Linux" elif [ ".$(egrep 'Red Hat Enterprise Linux' /etc/${rsource})" != . ]; then n="ed at nterprise inux" else @@ -703,6 +719,7 @@ __gather_linux_system_info() { slackware ) n="Slackware" ;; turbolinux ) n="TurboLinux" ;; unitedlinux ) n="UnitedLinux" ;; + oracle ) n="Oracle Linux" ;; system ) while read -r line; do [ "${n}x" != "systemx" ] && break @@ -714,8 +731,8 @@ __gather_linux_system_info() { done < /etc/${rsource} ;; os ) - nn=$(__unquote_string $(grep '^ID=' /etc/os-release | sed -e 's/^ID=\(.*\)$/\1/g')) - rv=$(__unquote_string $(grep '^VERSION_ID=' /etc/os-release | sed -e 's/^VERSION_ID=\(.*\)$/\1/g')) + nn="$(__unquote_string $(grep '^ID=' /etc/os-release | sed -e 's/^ID=\(.*\)$/\1/g'))" + rv="$(__unquote_string $(grep '^VERSION_ID=' /etc/os-release | sed -e 's/^VERSION_ID=\(.*\)$/\1/g'))" [ "${rv}x" != "x" ] && v=$(__parse_version_string "$rv") || v="" case $(echo ${nn} | tr '[:upper:]' '[:lower:]') in arch ) @@ -783,7 +800,7 @@ __gather_sunos_system_info() { DISTRO_NAME="Solaris" # Let's make sure we not actually on a Joyent's SmartOS VM since some releases # don't have SmartOS in `/etc/release`, only `Solaris` - $(uname -v | grep joyent >/dev/null 2>&1) + uname -v | grep joyent >/dev/null 2>&1 if [ $? -eq 0 ]; then DISTRO_NAME="SmartOS" fi @@ -1007,6 +1024,15 @@ __git_clone_and_checkout() { git fetch || return 1 # Tags are needed because of salt's versioning, also fetch that git fetch --tags || return 1 + + # If we have the SaltStack remote set as upstream, we also need to fetch the tags from there + if [ "x$(git remote -v | grep $_SALTSTACK_REPO_URL)" != "x" ]; then + git fetch --tags upstream + else + git remote add upstream $_SALTSTACK_REPO_URL + git fetch --tags upstream + fi + git reset --hard $GIT_REV || return 1 # Just calling `git reset --hard $GIT_REV` on a branch name that has @@ -1019,9 +1045,17 @@ __git_clone_and_checkout() { git pull --rebase || return 1 fi else - git clone $_SALTSTACK_REPO_URL || return 1 + git clone $_SALT_REPO_URL || return 1 cd $SALT_GIT_CHECKOUT_DIR + + if [ $_SALT_REPO_URL != $_SALTSTACK_REPO_URL ]; then + # We need to add the saltstack repository as a remote and fetch tags for proper versioning + git remote add upstream $_SALTSTACK_REPO_URL + git fetch --tags upstream + fi + git checkout $GIT_REV || return 1 + fi return 0 } @@ -1032,7 +1066,7 @@ __git_clone_and_checkout() { # DESCRIPTION: (DRY) apt-get install with noinput options #---------------------------------------------------------------------------------------------------------------------- __apt_get_install_noinput() { - apt-get install -y -o DPkg::Options::=--force-confold $@; return $? + apt-get install -y -o DPkg::Options::=--force-confold "${@}"; return $? } @@ -1041,7 +1075,7 @@ __apt_get_install_noinput() { # DESCRIPTION: (DRY) apt-get upgrade with noinput options #---------------------------------------------------------------------------------------------------------------------- __apt_get_upgrade_noinput() { - apt-get upgrade -y -o DPkg::Options::=--force-confold $@; return $? + apt-get upgrade -y -o DPkg::Options::=--force-confold; return $? } @@ -1473,7 +1507,12 @@ install_ubuntu_deps() { if [ $_START_DAEMONS -eq $BS_FALSE ]; then echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." fi + apt-get update + + # Install Keys + __apt_get_install_noinput debian-archive-keyring && apt-get update + if [ $DISTRO_MAJOR_VERSION -gt 12 ] || ([ $DISTRO_MAJOR_VERSION -eq 12 ] && [ $DISTRO_MINOR_VERSION -eq 10 ]); then # Above Ubuntu 12.04 add-apt-repository is in a different package __apt_get_install_noinput software-properties-common || return 1 @@ -1495,10 +1534,29 @@ install_ubuntu_deps() { # Minimal systems might not have upstart installed, install it __apt_get_install_noinput upstart - if [ $_INSTALL_CLOUD -eq $BS_TRUE ]; then - check_pip_allowed "You need to allow pip based installations(-P) in order to install apache-libcloud" + if [ $DISTRO_MAJOR_VERSION -gt 12 ] || ([ $DISTRO_MAJOR_VERSION -eq 12 ] && [ $DISTRO_MINOR_VERSION -gt 03 ]); then + __apt_get_install_noinput python-requests + __PIP_PACKAGES="" + else + check_pip_allowed "You need to allow pip based installations (-P) in order to install the python package 'requests'" __apt_get_install_noinput python-pip - pip install -U apache-libcloud>=$_LIBCLOUD_MIN_VERSION + __PIP_PACKAGES="requests" + pip install requests + fi + + # Additionally install procps and pciutils which allows for Docker boostraps. See 366#issuecomment-39666813 + __apt_get_install_noinput procps pciutils + + if [ $_INSTALL_CLOUD -eq $BS_TRUE ]; then + check_pip_allowed "You need to allow pip based installations (-P) in order to install 'apache-libcloud'" + if [ "x${__PIP_PACKAGES}" = "x" ]; then + __apt_get_install_noinput python-pip + fi + __PIP_PACKAGES="${__PIP_PACKAGES} 'apache-libcloud>=$_LIBCLOUD_MIN_VERSION'" + fi + + if [ "x${__PIP_PACKAGES}" != "x" ]; then + pip install -U ${__PIP_PACKAGES} fi if [ $_UPGRADE_SYS -eq $BS_TRUE ]; then @@ -1686,11 +1744,21 @@ install_debian_deps() { apt-get update + # Install Keys + __apt_get_install_noinput debian-archive-keyring && apt-get update + + # Both python-requests which is a hard dependency and apache-libcloud which is a soft dependency, under debian < 7 + # need to be installed using pip + check_pip_allowed "You need to allow pip based installations (-P) in order to install the python 'requests' package" + # Additionally install procps and pciutils which allows for Docker boostraps. See 366#issuecomment-39666813 + __apt_get_install_noinput python-pip procps pciutils + + __PIP_PACKAGES="requests" + if [ $_INSTALL_CLOUD -eq $BS_TRUE ]; then - check_pip_allowed "You need to allow pip based installations(-P) in order to install apache-libcloud" - __apt_get_install_noinput python-pip - pip install -U apache-libcloud>=$_LIBCLOUD_MIN_VERSION + __PIP_PACKAGES="${__PIP_PACKAGES} 'apache-libcloud>=$_LIBCLOUD_MIN_VERSION'" fi + pip install -U ${__PIP_PACKAGES} if [ $_UPGRADE_SYS -eq $BS_TRUE ]; then __apt_get_upgrade_noinput || return 1 @@ -1711,6 +1779,11 @@ install_debian_6_deps() { # No user interaction, libc6 restart services for example export DEBIAN_FRONTEND=noninteractive + apt-get update + + # Install Keys + __apt_get_install_noinput debian-archive-keyring && apt-get update + wget $_WGET_ARGS -q http://debian.saltstack.com/debian-salt-team-joehealy.gpg.key -O - | apt-key add - || return 1 if [ $_PIP_ALLOWED -eq $BS_TRUE ]; then @@ -1755,6 +1828,10 @@ _eof if [ "x$(grep -R 'backports.debian.org' /etc/apt)" = "x" ]; then echo "deb http://backports.debian.org/debian-backports squeeze-backports main" >> \ /etc/apt/sources.list.d/backports.list + + # Add the backports key + gpg --keyserver pgpkeys.mit.edu --recv-key 8B48AD6246925553 + gpg -a --export 8B48AD6246925553 | apt-key add - fi # Saltstack's Stable Debian repository @@ -1764,9 +1841,14 @@ _eof fi apt-get update || return 1 + # Python requests is available through Squeeze backports + # Additionally install procps and pciutils which allows for Docker boostraps. See 366#issuecomment-39666813 + __apt_get_install_noinput python-requests python-pip procps pciutils + if [ $_INSTALL_CLOUD -eq $BS_TRUE ]; then - check_pip_allowed "You need to allow pip based installations(-P) in order to install apache-libcloud" - pip install -U apache-libcloud>=$_LIBCLOUD_MIN_VERSION + check_pip_allowed "You need to allow pip based installations (-P) in order to install apache-libcloud" + __apt_get_install_noinput python-pip + pip install -U "apache-libcloud>=$_LIBCLOUD_MIN_VERSION" fi if [ $_UPGRADE_SYS -eq $BS_TRUE ]; then @@ -1790,6 +1872,10 @@ install_debian_7_deps() { # No user interaction, libc6 restart services for example export DEBIAN_FRONTEND=noninteractive + apt-get update + # Install Keys + __apt_get_install_noinput debian-archive-keyring && apt-get update + # Saltstack's Stable Debian repository if [ "x$(grep -R 'wheezy-saltstack' /etc/apt)" = "x" ]; then echo "deb http://debian.saltstack.com/debian wheezy-saltstack main" >> \ @@ -1824,15 +1910,22 @@ _eof apt-get update __apt_get_install_noinput -t unstable libzmq3 libzmq3-dev || return 1 - __apt_get_install_noinput build-essential python-dev python-pip || return 1 + __PACKAGES="build-essential python-dev python-pip python-requests" + # Additionally install procps and pciutils which allows for Docker boostraps. See 366#issuecomment-39666813 + __PACKAGES="${__PACKAGES} procps pciutils" + __apt_get_install_noinput ${__PACKAGES} || return 1 else apt-get update || return 1 - __apt_get_install_noinput python-zmq || return 1 + __PACKAGES="python-zmq python-requests" + # Additionally install procps and pciutils which allows for Docker boostraps. See 366#issuecomment-39666813 + __PACKAGES="${__PACKAGES} procps pciutils" + __apt_get_install_noinput ${__PACKAGES} || return 1 + fi if [ $_INSTALL_CLOUD -eq $BS_TRUE ]; then - check_pip_allowed "You need to allow pip based installations(-P) in order to install apache-libcloud" - pip install -U apache-libcloud>=$_LIBCLOUD_MIN_VERSION + check_pip_allowed "You need to allow pip based installations (-P) in order to install apache-libcloud" + pip install -U "apache-libcloud>=$_LIBCLOUD_MIN_VERSION" fi if [ $_UPGRADE_SYS -eq $BS_TRUE ]; then @@ -1860,6 +1953,10 @@ install_debian_git_deps() { export DEBIAN_FRONTEND=noninteractive apt-get update + + # Install Keys + __apt_get_install_noinput debian-archive-keyring && apt-get update + __apt_get_install_noinput lsb-release python python-pkg-resources python-crypto \ python-jinja2 python-m2crypto python-yaml msgpack-python python-pip \ git || return 1 @@ -1873,8 +1970,8 @@ install_debian_git_deps() { fi if [ $_INSTALL_CLOUD -eq $BS_TRUE ]; then - check_pip_allowed "You need to allow pip based installations(-P) in order to install apache-libcloud" - pip install -U apache-libcloud>=$_LIBCLOUD_MIN_VERSION + check_pip_allowed "You need to allow pip based installations (-P) in order to install apache-libcloud" + pip install -U "apache-libcloud>=$_LIBCLOUD_MIN_VERSION" fi if [ $_UPGRADE_SYS -eq $BS_TRUE ]; then @@ -2047,7 +2144,7 @@ install_debian_check_services() { # Fedora Install Functions # install_fedora_deps() { - packages="yum-utils PyYAML libyaml m2crypto python-crypto python-jinja2 python-msgpack python-zmq" + packages="yum-utils PyYAML libyaml m2crypto python-crypto python-jinja2 python-msgpack python-zmq python-requests" if [ $_INSTALL_CLOUD -eq $BS_TRUE ]; then packages="${packages} python-libcloud" @@ -2186,19 +2283,19 @@ install_centos_stable_deps() { yum -y update || return 1 fi - packages="yum-utils" + packages="yum-utils chkconfig" if [ $DISTRO_MAJOR_VERSION -eq 5 ]; then - packages="${packages} python26-PyYAML python26-m2crypto m2crypto python26 " + packages="${packages} python26-PyYAML python26-m2crypto m2crypto python26 python26-requests" packages="${packages} python26-crypto python26-msgpack python26-zmq python26-jinja2" if [ $_INSTALL_CLOUD -eq $BS_TRUE ]; then - check_pip_allowed "You need to allow pip based installations(-P) in order to install apache-libcloud" + check_pip_allowed "You need to allow pip based installations (-P) in order to install apache-libcloud" packages="${packages} python26-setuptools" fi else - packages="PyYAML m2crypto python-crypto python-msgpack python-zmq python-jinja2" + packages="${packages} PyYAML m2crypto python-crypto python-msgpack python-zmq python-jinja2 python-requests" if [ $_INSTALL_CLOUD -eq $BS_TRUE ]; then - check_pip_allowed "You need to allow pip based installations(-P) in order to install apache-libcloud" + check_pip_allowed "You need to allow pip based installations (-P) in order to install apache-libcloud" packages="${packages} python-pip" fi fi @@ -2206,11 +2303,11 @@ install_centos_stable_deps() { yum -y install ${packages} --enablerepo=${_EPEL_REPO} || return 1 if [ $_INSTALL_CLOUD -eq $BS_TRUE ]; then - check_pip_allowed "You need to allow pip based installations(-P) in order to install apache-libcloud" + check_pip_allowed "You need to allow pip based installations (-P) in order to install apache-libcloud" if [ $DISTRO_MAJOR_VERSION -eq 5 ]; then - easy_install-2.6 apache-libcloud>=$_LIBCLOUD_MIN_VERSION + easy_install-2.6 "apache-libcloud>=$_LIBCLOUD_MIN_VERSION" else - pip-python install apache-libcloud>=$_LIBCLOUD_MIN_VERSION + pip-python install "apache-libcloud>=$_LIBCLOUD_MIN_VERSION" fi fi @@ -2387,7 +2484,7 @@ install_red_hat_linux_stable_deps() { else OPTIONAL_ARCH=$CPU_ARCH_L fi - if [ $DISTRO_MAJOR_VERSION -eq 6 ] && [ "X$(rhn-channel -l | grep optional)" != "Xrhel-${OPTIONAL_ARCH}-server-optional-${DISTRO_MAJOR_VERSION}" ]; then + if [ $DISTRO_MAJOR_VERSION -eq 6 ] && case "X$(rhn-channel -l | grep optional)" in Xrhel-${OPTIONAL_ARCH}-server-optional-${DISTRO_MAJOR_VERSION}* ) false ;; * ) true ;; esac ; then echoerror "Failed to find RHN optional repo, please enable it using the GUI or rhn-channel command." return 1 fi @@ -2581,6 +2678,137 @@ install_red_hat_enterprise_workstation_testing_post() { # ####################################################################################################################### + +####################################################################################################################### +# +# Oracle Linux Install Functions +# +install_oracle_linux_stable_deps() { + install_centos_stable_deps || return 1 + return 0 +} + +install_oracle_linux_git_deps() { + install_centos_git_deps || return 1 + return 0 +} + +install_oracle_linux_testing_deps() { + install_centos_testing_deps || return 1 + return 0 +} + +install_oracle_linux_stable() { + install_centos_stable || return 1 + return 0 +} + +install_oracle_linux_git() { + install_centos_git || return 1 + return 0 +} + +install_oracle_linux_testing() { + install_centos_testing || return 1 + return 0 +} + +install_oracle_linux_stable_post() { + install_centos_stable_post || return 1 + return 0 +} + +install_oracle_linux_git_post() { + install_centos_git_post || return 1 + return 0 +} + + +install_oracle_linux_testing_post() { + install_centos_testing_post || return 1 + return 0 +} + +install_oracle_linux_restart_daemons() { + install_centos_restart_daemons || return 1 + return 0 +} + +install_oracle_linux_check_services() { + install_centos_check_services || return 1 + return 0 +} +# +# Ended Oracle Linux Install Functions +# +####################################################################################################################### + + +####################################################################################################################### +# +# Scientific Linux Install Functions +# +install_scientific_linux_stable_deps() { + install_centos_stable_deps || return 1 + return 0 +} + +install_scientific_linux_git_deps() { + install_centos_git_deps || return 1 + return 0 +} + +install_scientific_linux_testing_deps() { + install_centos_testing_deps || return 1 + return 0 +} + +install_scientific_linux_stable() { + install_centos_stable || return 1 + return 0 +} + +install_scientific_linux_git() { + install_centos_git || return 1 + return 0 +} + +install_scientific_linux_testing() { + install_centos_testing || return 1 + return 0 +} + +install_scientific_linux_stable_post() { + install_centos_stable_post || return 1 + return 0 +} + +install_scientific_linux_git_post() { + install_centos_git_post || return 1 + return 0 +} + + +install_scientific_linux_testing_post() { + install_centos_testing_post || return 1 + return 0 +} + +install_scientific_linux_restart_daemons() { + install_centos_restart_daemons || return 1 + return 0 +} + +install_scientific_linux_check_services() { + install_centos_check_services || return 1 + return 0 +} +# +# Ended Scientific Linux Install Functions +# +####################################################################################################################### + + ####################################################################################################################### # # Amazon Linux AMI Install Functions @@ -2599,18 +2827,18 @@ install_amazon_linux_ami_deps() { yum -y update || return 1 fi - packages="PyYAML m2crypto python-crypto python-msgpack python-zmq python-ordereddict python-jinja2" + packages="PyYAML m2crypto python-crypto python-msgpack python-zmq python-ordereddict python-jinja2 python-requests" if [ $_INSTALL_CLOUD -eq $BS_TRUE ]; then - check_pip_allowed "You need to allow pip based installations(-P) in order to install apache-libcloud" + check_pip_allowed "You need to allow pip based installations (-P) in order to install apache-libcloud" packages="${packages} python-pip" fi yum -y install ${packages} --enablerepo=${_EPEL_REPO} || return 1 if [ $_INSTALL_CLOUD -eq $BS_TRUE ]; then - check_pip_allowed "You need to allow pip based installations(-P) in order to install apache-libcloud" - pip-python install apache-libcloud>=$_LIBCLOUD_MIN_VERSION + check_pip_allowed "You need to allow pip based installations (-P) in order to install apache-libcloud" + pip-python install "apache-libcloud>=$_LIBCLOUD_MIN_VERSION" fi if [ "x${_EXTRA_PACKAGES}" != "x" ]; then @@ -2702,9 +2930,9 @@ install_arch_linux_git_deps() { pacman -Sy --noconfirm --needed pacman || return 1 # Don't fail if un-installing python2-distribute threw an error pacman -R --noconfirm --needed python2-distribute - pacman -Sy --noconfirm --needed git python2-crypto python2-setuptools \ - python2-jinja python2-m2crypto python2-markupsafe python2-msgpack \ - python2-psutil python2-yaml python2-pyzmq zeromq || return 1 + pacman -Sy --noconfirm --needed git python2-crypto python2-setuptools python2-jinja \ + python2-m2crypto python2-markupsafe python2-msgpack python2-psutil python2-yaml \ + python2-pyzmq zeromq python2-requests || return 1 __git_clone_and_checkout || return 1 @@ -2932,7 +3160,7 @@ config_freebsd_salt() { install_freebsd_git_deps() { install_freebsd_9_stable_deps || return 1 - /usr/local/sbin/pkg install -y git || return 1 + /usr/local/sbin/pkg install -y git www/py-requests || return 1 __git_clone_and_checkout || return 1 @@ -3077,8 +3305,7 @@ install_freebsd_restart_daemons() { # install_smartos_deps() { pkgin -y install \ - zeromq py27-m2crypto py27-crypto py27-msgpack py27-yaml \ - py27-jinja2 py27-zmq || return 1 + zeromq py27-m2crypto py27-crypto py27-msgpack py27-yaml py27-jinja2 py27-zmq py27-requests || return 1 # Let's trigger config_salt() if [ "$_TEMP_CONFIG_DIR" = "null" ]; then @@ -3213,7 +3440,7 @@ install_opensuse_stable_deps() { DISTRO_REPO="openSUSE_${DISTRO_MAJOR_VERSION}.${DISTRO_MINOR_VERSION}" # Is the repository already known - $(zypper repos | grep devel_languages_python >/dev/null 2>&1) + zypper repos | grep devel_languages_python >/dev/null 2>&1 if [ $? -eq 1 ]; then # zypper does not yet know nothing about devel_languages_python zypper --non-interactive addrepo --refresh \ @@ -3237,10 +3464,10 @@ install_opensuse_stable_deps() { zypper --gpg-auto-import-keys --non-interactive update || return 1 fi - packages="libzmq3 python python-Jinja2 python-M2Crypto python-PyYAML " + packages="libzmq3 python python-Jinja2 python-M2Crypto python-PyYAML python-requests" packages="${packages} python-msgpack-python python-pycrypto python-pyzmq python-xml" - if [ $_INSTALL_CLOUD -eq $BS_TRUE]; then + if [ $_INSTALL_CLOUD -eq $BS_TRUE ]; then packages="${packages} python-apache-libcloud" fi @@ -3386,7 +3613,7 @@ install_suse_11_stable_deps() { DISTRO_REPO="SLE_${DISTRO_MAJOR_VERSION}${DISTRO_PATCHLEVEL}" # Is the repository already known - $(zypper repos | grep devel_languages_python >/dev/null 2>&1) + zypper repos | grep devel_languages_python >/dev/null 2>&1 if [ $? -eq 1 ]; then # zypper does not yet know nothing about devel_languages_python zypper --non-interactive addrepo --refresh \ @@ -3400,7 +3627,7 @@ install_suse_11_stable_deps() { fi packages="libzmq3 python python-Jinja2 'python-M2Crypto>=0.21' python-msgpack-python" - packages="${packages} python-pycrypto python-pyzmq python-pip python-xml" + packages="${packages} python-pycrypto python-pyzmq python-pip python-xml python-requests" if [ $SUSE_PATCHLEVEL -eq 1 ]; then check_pip_allowed @@ -3548,9 +3775,9 @@ install_suse_check_services() { # __emerge() { if [ $_GENTOO_USE_BINHOST -eq $BS_TRUE ]; then - emerge --autounmask-write --getbinpkg $@; return $? + emerge --autounmask-write --getbinpkg "${@}"; return $? fi - emerge --autounmask-write $@; return $? + emerge --autounmask-write "${@}"; return $? } __gentoo_config_protection() { @@ -3587,11 +3814,12 @@ __gentoo_post_dep() { __gentoo_config_protection if [ $_INSTALL_CLOUD -eq $BS_TRUE ]; then - check_pip_allowed "You need to allow pip based installations(-P) in order to install apache-libcloud" + check_pip_allowed "You need to allow pip based installations (-P) in order to install apache-libcloud" __emerge -v 'dev-python/pip' - pip install -U apache-libcloud>=$_LIBCLOUD_MIN_VERSION + pip install -U "apache-libcloud>=$_LIBCLOUD_MIN_VERSION" fi + __emerge -vo 'dev-python/requests' __emerge -vo 'app-admin/salt' if [ "x${_EXTRA_PACKAGES}" != "x" ]; then @@ -3777,7 +4005,7 @@ preseed_master() { SEED_DEST="$_PKI_DIR/master/minions" [ -d $SEED_DEST ] || mkdir -p $SEED_DEST && chmod 700 $SEED_DEST || return 1 - for keyfile in $(ls $_TEMP_KEYS_DIR); do + for keyfile in $_TEMP_KEYS_DIR/*; do src_keyfile="${_TEMP_KEYS_DIR}/${keyfile}" dst_keyfile="${SEED_DEST}/${keyfile}" @@ -3814,11 +4042,11 @@ daemons_running() { if [ "${DISTRO_NAME}" = "SmartOS" ]; then if [ "$(svcs -Ho STA salt-$fname)" != "ON" ]; then echoerror "salt-$fname was not found running" - FAILED_DAEMONS=$(expr $FAILED_DAEMONS + 1) + FAILED_DAEMONS=$((FAILED_DAEMONS + 1)) fi elif [ "x$(ps wwwaux | grep -v grep | grep salt-$fname)" = "x" ]; then echoerror "salt-$fname was not found running" - FAILED_DAEMONS=$(expr $FAILED_DAEMONS + 1) + FAILED_DAEMONS=$((FAILED_DAEMONS + 1)) fi done return $FAILED_DAEMONS @@ -4054,6 +4282,12 @@ master: $_SALT_MASTER_ADDRESS _eof fi +# Drop the minion id if passed +if [ $_SALT_MINION_ID != "null" ]; then + [ ! -d $_SALT_ETC_DIR ] && mkdir -p $_SALT_ETC_DIR + echo $_SALT_MINION_ID > $_SALT_ETC_DIR/minion_id +fi + # Run any post install function. Only execute function if not in config mode only if [ $_CONFIG_ONLY -eq $BS_FALSE ] && [ "$POST_INSTALL_FUNC" != "null" ]; then echoinfo "Running ${POST_INSTALL_FUNC}()" diff --git a/tests/bootstrap/ext/nb_popen.py b/tests/bootstrap/ext/nb_popen.py new file mode 100644 index 0000000..6b32585 --- /dev/null +++ b/tests/bootstrap/ext/nb_popen.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)` + :copyright: © 2013 by the SaltStack Team, see AUTHORS for more details. + :license: Apache 2.0, see LICENSE for more details. + + + salt.utils.nb_popen + ~~~~~~~~~~~~~~~~~~~ + + Non blocking subprocess Popen. + + This functionality has been adapted to work on windows following the recipe + found on: + + http://code.activestate.com/recipes/440554/ +''' + +# Import python libs +import os +import sys +import time +import errno +import select +import logging +import tempfile +import subprocess + +if subprocess.mswindows: + from win32file import ReadFile, WriteFile + from win32pipe import PeekNamedPipe + import msvcrt +else: + import fcntl + +log = logging.getLogger(__name__) + + +class NonBlockingPopen(subprocess.Popen): + + #_stdin_logger_name_ = 'salt.utils.nb_popen.STDIN.PID-{pid}' + _stdout_logger_name_ = 'salt.utils.nb_popen.STDOUT.PID-{pid}' + _stderr_logger_name_ = 'salt.utils.nb_popen.STDERR.PID-{pid}' + + def __init__(self, *args, **kwargs): + self.stream_stds = kwargs.pop('stream_stds', False) + + # Half a megabyte in memory is more than enough to start writing to + # a temporary file. + self.max_size_in_mem = kwargs.pop('max_size_in_mem', 512000) + + # Let's configure the std{in, out,err} logging handler names + #self._stdin_logger_name_ = kwargs.pop( + # 'stdin_logger_name', self._stdin_logger_name_ + #) + self._stdout_logger_name_ = kwargs.pop( + 'stdout_logger_name', self._stdout_logger_name_ + ) + self._stderr_logger_name_ = kwargs.pop( + 'stderr_logger_name', self._stderr_logger_name_ + ) + + stderr = kwargs.get('stderr', None) + + super(NonBlockingPopen, self).__init__(*args, **kwargs) + + #self._stdin_logger = logging.getLogger( + # self._stdin_logger_name_.format(pid=self.pid) + #) + + self.stdout_buff = tempfile.SpooledTemporaryFile(self.max_size_in_mem) + self._stdout_logger = logging.getLogger( + self._stdout_logger_name_.format(pid=self.pid) + ) + + if stderr is subprocess.STDOUT: + self.stderr_buff = self.stdout_buff + self._stderr_logger = self._stdout_logger + else: + self.stderr_buff = tempfile.SpooledTemporaryFile( + self.max_size_in_mem + ) + self._stderr_logger = logging.getLogger( + self._stderr_logger_name_.format(pid=self.pid) + ) + + self._stderr_logger = logging.getLogger( + self._stderr_logger_name_.format(pid=self.pid) + ) + + log.info( + 'Running command under pid {0}: {1!r}'.format(self.pid, *args) + ) + + def recv(self, maxsize=None): + return self._recv('stdout', maxsize) + + def recv_err(self, maxsize=None): + return self._recv('stderr', maxsize) + + def send_recv(self, input='', maxsize=None): + return self.send(input), self.recv(maxsize), self.recv_err(maxsize) + + def get_conn_maxsize(self, which, maxsize): + if maxsize is None: + maxsize = 1024 + elif maxsize < 1: + maxsize = 1 + return getattr(self, which), maxsize + + def _close(self, which): + getattr(self, which).close() + setattr(self, which, None) + + if subprocess.mswindows: + def send(self, input): + if not self.stdin: + return None + + try: + x = msvcrt.get_osfhandle(self.stdin.fileno()) + (errCode, written) = WriteFile(x, input) + #self._stdin_logger.debug(input.rstrip()) + except ValueError: + return self._close('stdin') + except (subprocess.pywintypes.error, Exception) as why: + if why[0] in (109, errno.ESHUTDOWN): + return self._close('stdin') + raise + + return written + + def _recv(self, which, maxsize): + conn, maxsize = self.get_conn_maxsize(which, maxsize) + if conn is None: + return None + + try: + x = msvcrt.get_osfhandle(conn.fileno()) + (read, nAvail, nMessage) = PeekNamedPipe(x, 0) + if maxsize < nAvail: + nAvail = maxsize + if nAvail > 0: + (errCode, read) = ReadFile(x, nAvail, None) + except ValueError: + return self._close(which) + except (subprocess.pywintypes.error, Exception) as why: + if why[0] in (109, errno.ESHUTDOWN): + return self._close(which) + raise + + getattr(self, '{0}_buff'.format(which)).write(read) + getattr(self, '_{0}_logger'.format(which)).debug(read.rstrip()) + if self.stream_stds: + getattr(sys, which).write(read) + + if self.universal_newlines: + read = self._translate_newlines(read) + return read + + else: + + def send(self, input): + if not self.stdin: + return None + + if not select.select([], [self.stdin], [], 0)[1]: + return 0 + + try: + written = os.write(self.stdin.fileno(), input) + #self._stdin_logger.debug(input.rstrip()) + except OSError as why: + if why[0] == errno.EPIPE: # broken pipe + return self._close('stdin') + raise + + return written + + def _recv(self, which, maxsize): + conn, maxsize = self.get_conn_maxsize(which, maxsize) + if conn is None: + return None + + flags = fcntl.fcntl(conn, fcntl.F_GETFL) + if not conn.closed: + fcntl.fcntl(conn, fcntl.F_SETFL, flags | os.O_NONBLOCK) + + try: + if not select.select([conn], [], [], 0)[0]: + return '' + + buff = conn.read(maxsize) + if not buff: + return self._close(which) + + if self.universal_newlines: + buff = self._translate_newlines(buff) + + getattr(self, '{0}_buff'.format(which)).write(buff) + getattr(self, '_{0}_logger'.format(which)).debug(buff.rstrip()) + if self.stream_stds: + getattr(sys, which).write(buff) + + return buff + finally: + if not conn.closed: + fcntl.fcntl(conn, fcntl.F_SETFL, flags) + + def poll_and_read_until_finish(self): + silent_iterations = 0 + while self.poll() is None: + if self.stdout is not None: + silent_iterations = 0 + self.recv() + + if self.stderr is not None: + silent_iterations = 0 + self.recv_err() + + silent_iterations += 1 + + if silent_iterations > 100: + silent_iterations = 0 + (stdoutdata, stderrdata) = self.communicate() + if stdoutdata: + log.debug(stdoutdata) + if stderrdata: + log.error(stderrdata) + time.sleep(0.01) + + def communicate(self, input=None): + super(NonBlockingPopen, self).communicate(input) + self.stdout_buff.flush() + self.stdout_buff.seek(0) + self.stderr_buff.flush() + self.stderr_buff.seek(0) + return self.stdout_buff.read(), self.stderr_buff.read() diff --git a/tests/bootstrap/test_lint.py b/tests/bootstrap/test_lint.py index adc233b..a05d0ca 100644 --- a/tests/bootstrap/test_lint.py +++ b/tests/bootstrap/test_lint.py @@ -22,6 +22,8 @@ class LintTestCase(BootstrapTestCase): 0, self.run_script( script=os.path.join(EXT_DIR, 'checkbashisms'), - args=('-pxfn', BOOTSTRAP_SCRIPT_PATH) + args=('-pxfn', BOOTSTRAP_SCRIPT_PATH), + timeout=120, + stream_stds=True ) ) diff --git a/tests/bootstrap/unittesting.py b/tests/bootstrap/unittesting.py index 4204057..7027bc9 100644 --- a/tests/bootstrap/unittesting.py +++ b/tests/bootstrap/unittesting.py @@ -15,10 +15,11 @@ import os import sys import fcntl import signal +import logging import subprocess from datetime import datetime, timedelta -# Import salt bootstrap libs +# Import salt testing libs from salttesting import * from salttesting.ext.os_data import GRAINS @@ -30,8 +31,12 @@ BOOTSTRAP_SCRIPT_PATH = os.path.join(PARENT_DIR, 'bootstrap-salt.sh') class NonBlockingPopen(subprocess.Popen): + _stdout_logger_name_ = 'salt-bootstrap.NonBlockingPopen.STDOUT.PID-{pid}' + _stderr_logger_name_ = 'salt-bootstrap.NonBlockingPopen.STDERR.PID-{pid}' + def __init__(self, *args, **kwargs): self.stream_stds = kwargs.pop('stream_stds', False) + self._stdout_logger = self._stderr_logger = None super(NonBlockingPopen, self).__init__(*args, **kwargs) if self.stdout is not None and self.stream_stds: fod = self.stdout.fileno() @@ -46,12 +51,22 @@ class NonBlockingPopen(subprocess.Popen): self.ebuff = '' def poll(self): + if self._stdout_logger is None: + self._stdout_logger = logging.getLogger( + self._stdout_logger_name_.format(pid=self.pid) + ) + if self._stderr_logger is None: + self._stderr_logger = logging.getLogger( + self._stderr_logger_name_.format(pid=self.pid) + ) poll = super(NonBlockingPopen, self).poll() if self.stdout is not None and self.stream_stds: try: obuff = self.stdout.read() self.obuff += obuff + if obuff.strip(): + self._stdout_logger.info(obuff.strip()) sys.stdout.write(obuff) except IOError, err: if err.errno not in (11, 35): @@ -62,6 +77,8 @@ class NonBlockingPopen(subprocess.Popen): try: ebuff = self.stderr.read() self.ebuff += ebuff + if ebuff.strip(): + self._stderr_logger.info(ebuff.strip()) sys.stderr.write(ebuff) except IOError, err: if err.errno not in (11, 35): @@ -103,8 +120,6 @@ class BootstrapTestCase(TestCase): cmd = [script] + list(args) - outbuff = errbuff = '' - popen_kwargs = { 'cwd': cwd, 'shell': True, diff --git a/tests/runtests.py b/tests/runtests.py index 9c4dfa7..7125195 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -68,7 +68,8 @@ def main(): parser = BootstrapSuiteParser( TEST_DIR, xml_output_dir=XML_OUTPUT_DIR, - html_output_dir=HTML_OUTPUT_DIR + html_output_dir=HTML_OUTPUT_DIR, + tests_logfile=os.path.join(tempfile.gettempdir(), 'bootstrap-runtests.log') ) options, _ = parser.parse_args()