Merge branch 'master' into 2019.2.1-merge-master

.ci/docs

@@ -2,7 +2,7 @@
 
 // Define the maximum time, in hours, that a test run should run for
 def global_timeout = 2
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 
 properties([
     buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')),

.ci/kitchen-amazon1-py2

@@ -0,0 +1,159 @@
+@Library('salt@1.1') _
+
+// Define the maximum time, in hours, that a test run should run for
+def testrun_timeout = 6
+// Now define a global pipeline timeout. This is the test run timeout with one(1) additional
+// hour to allow for artifacts to be downloaded, if possible.
+def global_timeout = testrun_timeout + 1;
+
+def distro_name = 'amazon'
+def distro_version = '1'
+def python_version = 'py2'
+def test_transport = 'ZeroMQ'
+def salt_target_branch = 'master'
+def golden_images_branch = '2019.2'
+def nox_passthrough_opts = '--ssh-tests'
+
+properties([
+    buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')),
+    parameters([
+        booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull')
+    ])
+])
+
+// Be sure to cancel any previously running builds
+def buildNumber = env.BUILD_NUMBER as int
+if (buildNumber > 1) {
+    // This will cancel the previous build which also defined a matching milestone
+    milestone(buildNumber - 1)
+}
+// Define a milestone for this build so that, if another build starts, this one will be aborted
+milestone(buildNumber)
+
+
+wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
+    withEnv([
+        'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml',
+        'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml',
+        'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml',
+        "NOX_ENV_NAME=runtests-${test_transport.toLowerCase()}",
+        'NOX_ENABLE_FROM_FILENAMES=true',
+        "NOX_PASSTHROUGH_OPTS=${nox_passthrough_opts}",
+        "SALT_TARGET_BRANCH=${salt_target_branch}",
+        "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
+        "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
+        'RBENV_VERSION=2.6.3',
+        "TEST_SUITE=${python_version}",
+        "TEST_PLATFORM=${distro_name}-${distro_version}",
+        "TEST_TRANSPORT=${test_transport}",
+        "FORCE_FULL=${params.runFull}",
+    ]) {
+        // Checkout the repo
+        stage('Clone') {
+            cleanWs notFailBuild: true
+            checkout scm
+            sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}'
+        }
+
+        // Setup the kitchen required bundle
+        stage('Setup') {
+            sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant'
+        }
+
+        stage('Create VM') {
+            retry(3) {
+                sh '''
+                t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
+                '''
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-create.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-create.log"
+                fi
+                """
+            }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
+        }
+
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
+                    fi
+                    """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
+                    }
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
+                        sh '''
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
+                        '''
+                    }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+// vim: ft=groovy

.ci/kitchen-amazon2-py2

@@ -10,7 +10,7 @@ def distro_name = 'amazon'
 def distro_version = '2'
 def python_version = 'py2'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-amazon2-py3

@@ -10,7 +10,7 @@ def distro_name = 'amazon'
 def distro_version = '2'
 def python_version = 'py3'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-archlts-py2

@@ -0,0 +1,159 @@
+@Library('salt@1.1') _
+
+// Define the maximum time, in hours, that a test run should run for
+def testrun_timeout = 6
+// Now define a global pipeline timeout. This is the test run timeout with one(1) additional
+// hour to allow for artifacts to be downloaded, if possible.
+def global_timeout = testrun_timeout + 1;
+
+def distro_name = 'arch'
+def distro_version = 'lts'
+def python_version = 'py2'
+def test_transport = 'ZeroMQ'
+def salt_target_branch = 'master'
+def golden_images_branch = '2019.2'
+def nox_passthrough_opts = '-n integration.modules.test_pkg'
+
+properties([
+    buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')),
+    parameters([
+        booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull')
+    ])
+])
+
+// Be sure to cancel any previously running builds
+def buildNumber = env.BUILD_NUMBER as int
+if (buildNumber > 1) {
+    // This will cancel the previous build which also defined a matching milestone
+    milestone(buildNumber - 1)
+}
+// Define a milestone for this build so that, if another build starts, this one will be aborted
+milestone(buildNumber)
+
+
+wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
+    withEnv([
+        'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml',
+        'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml',
+        'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml',
+        "NOX_ENV_NAME=runtests-${test_transport.toLowerCase()}",
+        'NOX_ENABLE_FROM_FILENAMES=true',
+        "NOX_PASSTHROUGH_OPTS=${nox_passthrough_opts}",
+        "SALT_TARGET_BRANCH=${salt_target_branch}",
+        "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
+        "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
+        'RBENV_VERSION=2.6.3',
+        "TEST_SUITE=${python_version}",
+        "TEST_PLATFORM=${distro_name}-${distro_version}",
+        "TEST_TRANSPORT=${test_transport}",
+        "FORCE_FULL=${params.runFull}",
+    ]) {
+        // Checkout the repo
+        stage('Clone') {
+            cleanWs notFailBuild: true
+            checkout scm
+            sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}'
+        }
+
+        // Setup the kitchen required bundle
+        stage('Setup') {
+            sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant'
+        }
+
+        stage('Create VM') {
+            retry(3) {
+                sh '''
+                t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
+                '''
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-create.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-create.log"
+                fi
+                """
+            }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
+        }
+
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
+                    fi
+                    """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
+                    }
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
+                        sh '''
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
+                        '''
+                    }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+// vim: ft=groovy

.ci/kitchen-archlts-py3

@@ -0,0 +1,159 @@
+@Library('salt@1.1') _
+
+// Define the maximum time, in hours, that a test run should run for
+def testrun_timeout = 6
+// Now define a global pipeline timeout. This is the test run timeout with one(1) additional
+// hour to allow for artifacts to be downloaded, if possible.
+def global_timeout = testrun_timeout + 1;
+
+def distro_name = 'arch'
+def distro_version = 'lts'
+def python_version = 'py3'
+def test_transport = 'ZeroMQ'
+def salt_target_branch = 'master'
+def golden_images_branch = '2019.2'
+def nox_passthrough_opts = '-n integration.modules.test_pkg'
+
+properties([
+    buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')),
+    parameters([
+        booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull')
+    ])
+])
+
+// Be sure to cancel any previously running builds
+def buildNumber = env.BUILD_NUMBER as int
+if (buildNumber > 1) {
+    // This will cancel the previous build which also defined a matching milestone
+    milestone(buildNumber - 1)
+}
+// Define a milestone for this build so that, if another build starts, this one will be aborted
+milestone(buildNumber)
+
+
+wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
+    withEnv([
+        'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml',
+        'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml',
+        'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml',
+        "NOX_ENV_NAME=runtests-${test_transport.toLowerCase()}",
+        'NOX_ENABLE_FROM_FILENAMES=true',
+        "NOX_PASSTHROUGH_OPTS=${nox_passthrough_opts}",
+        "SALT_TARGET_BRANCH=${salt_target_branch}",
+        "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
+        "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
+        'RBENV_VERSION=2.6.3',
+        "TEST_SUITE=${python_version}",
+        "TEST_PLATFORM=${distro_name}-${distro_version}",
+        "TEST_TRANSPORT=${test_transport}",
+        "FORCE_FULL=${params.runFull}",
+    ]) {
+        // Checkout the repo
+        stage('Clone') {
+            cleanWs notFailBuild: true
+            checkout scm
+            sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}'
+        }
+
+        // Setup the kitchen required bundle
+        stage('Setup') {
+            sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant'
+        }
+
+        stage('Create VM') {
+            retry(3) {
+                sh '''
+                t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
+                '''
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-create.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-create.log"
+                fi
+                """
+            }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
+        }
+
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
+                    fi
+                    """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
+                    }
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
+                        sh '''
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
+                        '''
+                    }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+// vim: ft=groovy

.ci/kitchen-centos6-py2

@@ -10,7 +10,7 @@ def distro_name = 'centos'
 def distro_version = '6'
 def python_version = 'py2'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-centos7-py2

@@ -10,7 +10,7 @@ def distro_name = 'centos'
 def distro_version = '7'
 def python_version = 'py2'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-centos7-py2-tcp

@@ -10,7 +10,7 @@ def distro_name = 'centos'
 def distro_version = '7'
 def python_version = 'py2'
 def test_transport = 'TCP'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-centos7-py3

@@ -10,7 +10,7 @@ def distro_name = 'centos'
 def distro_version = '7'
 def python_version = 'py3'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-centos7-py3-tcp

@@ -10,7 +10,7 @@ def distro_name = 'centos'
 def distro_version = '7'
 def python_version = 'py3'
 def test_transport = 'TCP'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-debian10-py3

@@ -0,0 +1,159 @@
+@Library('salt@1.1') _
+
+// Define the maximum time, in hours, that a test run should run for
+def testrun_timeout = 6
+// Now define a global pipeline timeout. This is the test run timeout with one(1) additional
+// hour to allow for artifacts to be downloaded, if possible.
+def global_timeout = testrun_timeout + 1;
+
+def distro_name = 'debian'
+def distro_version = '10'
+def python_version = 'py3'
+def test_transport = 'ZeroMQ'
+def salt_target_branch = 'master'
+def golden_images_branch = '2019.2'
+def nox_passthrough_opts = '--ssh-tests'
+
+properties([
+    buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')),
+    parameters([
+        booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull')
+    ])
+])
+
+// Be sure to cancel any previously running builds
+def buildNumber = env.BUILD_NUMBER as int
+if (buildNumber > 1) {
+    // This will cancel the previous build which also defined a matching milestone
+    milestone(buildNumber - 1)
+}
+// Define a milestone for this build so that, if another build starts, this one will be aborted
+milestone(buildNumber)
+
+
+wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
+    withEnv([
+        'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml',
+        'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml',
+        'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml',
+        "NOX_ENV_NAME=runtests-${test_transport.toLowerCase()}",
+        'NOX_ENABLE_FROM_FILENAMES=true',
+        "NOX_PASSTHROUGH_OPTS=${nox_passthrough_opts}",
+        "SALT_TARGET_BRANCH=${salt_target_branch}",
+        "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
+        "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
+        'RBENV_VERSION=2.6.3',
+        "TEST_SUITE=${python_version}",
+        "TEST_PLATFORM=${distro_name}-${distro_version}",
+        "TEST_TRANSPORT=${test_transport}",
+        "FORCE_FULL=${params.runFull}",
+    ]) {
+        // Checkout the repo
+        stage('Clone') {
+            cleanWs notFailBuild: true
+            checkout scm
+            sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}'
+        }
+
+        // Setup the kitchen required bundle
+        stage('Setup') {
+            sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant'
+        }
+
+        stage('Create VM') {
+            retry(3) {
+                sh '''
+                t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
+                '''
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-create.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-create.log"
+                fi
+                """
+            }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
+        }
+
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
+                    fi
+                    """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
+                    }
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
+                        sh '''
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
+                        '''
+                    }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+// vim: ft=groovy

.ci/kitchen-debian8-py2

@@ -10,7 +10,7 @@ def distro_name = 'debian'
 def distro_version = '8'
 def python_version = 'py2'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-debian9-py2

@@ -10,7 +10,7 @@ def distro_name = 'debian'
 def distro_version = '9'
 def python_version = 'py2'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-debian9-py3

@@ -10,7 +10,7 @@ def distro_name = 'debian'
 def distro_version = '9'
 def python_version = 'py3'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-fedora29-py2

@@ -10,7 +10,7 @@ def distro_name = 'fedora'
 def distro_version = '29'
 def python_version = 'py2'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-fedora29-py3

@@ -10,7 +10,7 @@ def distro_name = 'fedora'
 def distro_version = '29'
 def python_version = 'py3'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-fedora30-py2

@@ -0,0 +1,159 @@
+@Library('salt@1.1') _
+
+// Define the maximum time, in hours, that a test run should run for
+def testrun_timeout = 6
+// Now define a global pipeline timeout. This is the test run timeout with one(1) additional
+// hour to allow for artifacts to be downloaded, if possible.
+def global_timeout = testrun_timeout + 1;
+
+def distro_name = 'fedora'
+def distro_version = '30'
+def python_version = 'py2'
+def test_transport = 'ZeroMQ'
+def salt_target_branch = 'master'
+def golden_images_branch = '2019.2'
+def nox_passthrough_opts = '--ssh-tests'
+
+properties([
+    buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')),
+    parameters([
+        booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull')
+    ])
+])
+
+// Be sure to cancel any previously running builds
+def buildNumber = env.BUILD_NUMBER as int
+if (buildNumber > 1) {
+    // This will cancel the previous build which also defined a matching milestone
+    milestone(buildNumber - 1)
+}
+// Define a milestone for this build so that, if another build starts, this one will be aborted
+milestone(buildNumber)
+
+
+wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
+    withEnv([
+        'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml',
+        'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml',
+        'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml',
+        "NOX_ENV_NAME=runtests-${test_transport.toLowerCase()}",
+        'NOX_ENABLE_FROM_FILENAMES=true',
+        "NOX_PASSTHROUGH_OPTS=${nox_passthrough_opts}",
+        "SALT_TARGET_BRANCH=${salt_target_branch}",
+        "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
+        "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
+        'RBENV_VERSION=2.6.3',
+        "TEST_SUITE=${python_version}",
+        "TEST_PLATFORM=${distro_name}-${distro_version}",
+        "TEST_TRANSPORT=${test_transport}",
+        "FORCE_FULL=${params.runFull}",
+    ]) {
+        // Checkout the repo
+        stage('Clone') {
+            cleanWs notFailBuild: true
+            checkout scm
+            sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}'
+        }
+
+        // Setup the kitchen required bundle
+        stage('Setup') {
+            sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant'
+        }
+
+        stage('Create VM') {
+            retry(3) {
+                sh '''
+                t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
+                '''
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-create.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-create.log"
+                fi
+                """
+            }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
+        }
+
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
+                    fi
+                    """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
+                    }
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
+                        sh '''
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
+                        '''
+                    }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+// vim: ft=groovy

.ci/kitchen-fedora30-py3

@@ -0,0 +1,159 @@
+@Library('salt@1.1') _
+
+// Define the maximum time, in hours, that a test run should run for
+def testrun_timeout = 6
+// Now define a global pipeline timeout. This is the test run timeout with one(1) additional
+// hour to allow for artifacts to be downloaded, if possible.
+def global_timeout = testrun_timeout + 1;
+
+def distro_name = 'fedora'
+def distro_version = '30'
+def python_version = 'py3'
+def test_transport = 'ZeroMQ'
+def salt_target_branch = 'master'
+def golden_images_branch = '2019.2'
+def nox_passthrough_opts = '--ssh-tests'
+
+properties([
+    buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')),
+    parameters([
+        booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull')
+    ])
+])
+
+// Be sure to cancel any previously running builds
+def buildNumber = env.BUILD_NUMBER as int
+if (buildNumber > 1) {
+    // This will cancel the previous build which also defined a matching milestone
+    milestone(buildNumber - 1)
+}
+// Define a milestone for this build so that, if another build starts, this one will be aborted
+milestone(buildNumber)
+
+
+wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
+    withEnv([
+        'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml',
+        'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml',
+        'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml',
+        "NOX_ENV_NAME=runtests-${test_transport.toLowerCase()}",
+        'NOX_ENABLE_FROM_FILENAMES=true',
+        "NOX_PASSTHROUGH_OPTS=${nox_passthrough_opts}",
+        "SALT_TARGET_BRANCH=${salt_target_branch}",
+        "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
+        "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
+        'RBENV_VERSION=2.6.3',
+        "TEST_SUITE=${python_version}",
+        "TEST_PLATFORM=${distro_name}-${distro_version}",
+        "TEST_TRANSPORT=${test_transport}",
+        "FORCE_FULL=${params.runFull}",
+    ]) {
+        // Checkout the repo
+        stage('Clone') {
+            cleanWs notFailBuild: true
+            checkout scm
+            sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}'
+        }
+
+        // Setup the kitchen required bundle
+        stage('Setup') {
+            sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant'
+        }
+
+        stage('Create VM') {
+            retry(3) {
+                sh '''
+                t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
+                '''
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-create.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-create.log"
+                fi
+                """
+            }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
+        }
+
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
+                    fi
+                    """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
+                    }
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
+                        sh '''
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
+                        '''
+                    }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+// vim: ft=groovy

.ci/kitchen-macosxmojave-py2

@@ -0,0 +1,146 @@
+@Library('salt@1.1') _
+
+// Define the maximum time, in hours, that a test run should run for
+def testrun_timeout = 6
+// Now define a global pipeline timeout. This is the test run timeout with one(1) additional
+// hour to allow for artifacts to be downloaded, if possible.
+def global_timeout = testrun_timeout + 1;
+
+def distro_name = 'macosx'
+def distro_version = 'mojave'
+def python_version = 'py2'
+def test_transport = 'ZeroMQ'
+def salt_target_branch = 'master'
+def golden_images_branch = '2019.2'
+def nox_passthrough_opts = ''
+
+properties([
+    buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')),
+    parameters([
+        booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull')
+    ])
+])
+
+// Be sure to cancel any previously running builds
+def buildNumber = env.BUILD_NUMBER as int
+if (buildNumber > 1) {
+    // This will cancel the previous build which also defined a matching milestone
+    milestone(buildNumber - 1)
+}
+// Define a milestone for this build so that, if another build starts, this one will be aborted
+milestone(buildNumber)
+
+wrappedNode('kitchen-slave-mac', global_timeout, '#jenkins-prod-pr') {
+    withEnv([
+        'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml',
+        'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/verifier.yml',
+        'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml',
+        "NOX_ENV_NAME=runtests-${test_transport.toLowerCase()}",
+        'NOX_ENABLE_FROM_FILENAMES=true',
+        "NOX_PASSTHROUGH_OPTS=${nox_passthrough_opts}",
+        "SALT_TARGET_BRANCH=${salt_target_branch}",
+        "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
+        "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
+        "PATH=/Users/parallels/.rbenv/shims:/Users/parallels/.rbenv/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/salt/bin:/usr/local/sbin",
+        'RBENV_VERSION=2.6.3',
+        "TEST_SUITE=${python_version}",
+        "TEST_PLATFORM=${distro_name}-${distro_version}",
+        "TEST_TRANSPORT=${test_transport}",
+        "FORCE_FULL=${params.runFull}",
+    ]) {
+        stage('VM Cleanup') {
+            sh '''
+            for i in `prlctl list -aij|jq -r '.[]|select((.Uptime|tonumber > 86400) and (.State == "running"))|.ID'`
+            do
+                prlctl stop $i --kill
+            done
+            # don't delete vm's that haven't started yet ((.State == "stopped") and (.Uptime == "0"))
+            for i in `prlctl list -aij|jq -r '.[]|select((.Uptime|tonumber > 0) and (.State != "running"))|.ID'`
+            do
+                prlctl delete $i
+            done
+            '''
+        }
+        // Checkout the repo
+        stage('Clone') {
+            cleanWs notFailBuild: true
+            checkout scm
+            sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}'
+        }
+
+        // Setup the kitchen required bundle
+        stage('Setup') {
+            sh 'bundle install --with vagrant macos --without ec2 windows opennebula docker'
+        }
+
+        stage('Create VM') {
+            sh '''
+            bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+            '''
+            sh """
+            if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-create.log"
+            fi
+            if [ -s ".kitchen/logs/kitchen.log" ]; then
+                mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-create.log"
+            fi
+            """
+        }
+
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.vagrant.d/insecure_private_key; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
+                    fi
+                    """
+                }
+                stage('Run Tests') {
+                    sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+// vim: ft=groovy

.ci/kitchen-macosxmojave-py3

@@ -0,0 +1,146 @@
+@Library('salt@1.1') _
+
+// Define the maximum time, in hours, that a test run should run for
+def testrun_timeout = 6
+// Now define a global pipeline timeout. This is the test run timeout with one(1) additional
+// hour to allow for artifacts to be downloaded, if possible.
+def global_timeout = testrun_timeout + 1;
+
+def distro_name = 'macosx'
+def distro_version = 'mojave'
+def python_version = 'py3'
+def test_transport = 'ZeroMQ'
+def salt_target_branch = 'master'
+def golden_images_branch = '2019.2'
+def nox_passthrough_opts = ''
+
+properties([
+    buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')),
+    parameters([
+        booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull')
+    ])
+])
+
+// Be sure to cancel any previously running builds
+def buildNumber = env.BUILD_NUMBER as int
+if (buildNumber > 1) {
+    // This will cancel the previous build which also defined a matching milestone
+    milestone(buildNumber - 1)
+}
+// Define a milestone for this build so that, if another build starts, this one will be aborted
+milestone(buildNumber)
+
+wrappedNode('kitchen-slave-mac', global_timeout, '#jenkins-prod-pr') {
+    withEnv([
+        'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml',
+        'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/verifier.yml',
+        'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml',
+        "NOX_ENV_NAME=runtests-${test_transport.toLowerCase()}",
+        'NOX_ENABLE_FROM_FILENAMES=true',
+        "NOX_PASSTHROUGH_OPTS=${nox_passthrough_opts}",
+        "SALT_TARGET_BRANCH=${salt_target_branch}",
+        "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
+        "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
+        "PATH=/Users/parallels/.rbenv/shims:/Users/parallels/.rbenv/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/salt/bin:/usr/local/sbin",
+        'RBENV_VERSION=2.6.3',
+        "TEST_SUITE=${python_version}",
+        "TEST_PLATFORM=${distro_name}-${distro_version}",
+        "TEST_TRANSPORT=${test_transport}",
+        "FORCE_FULL=${params.runFull}",
+    ]) {
+        stage('VM Cleanup') {
+            sh '''
+            for i in `prlctl list -aij|jq -r '.[]|select((.Uptime|tonumber > 86400) and (.State == "running"))|.ID'`
+            do
+                prlctl stop $i --kill
+            done
+            # don't delete vm's that haven't started yet ((.State == "stopped") and (.Uptime == "0"))
+            for i in `prlctl list -aij|jq -r '.[]|select((.Uptime|tonumber > 0) and (.State != "running"))|.ID'`
+            do
+                prlctl delete $i
+            done
+            '''
+        }
+        // Checkout the repo
+        stage('Clone') {
+            cleanWs notFailBuild: true
+            checkout scm
+            sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}'
+        }
+
+        // Setup the kitchen required bundle
+        stage('Setup') {
+            sh 'bundle install --with vagrant macos --without ec2 windows opennebula docker'
+        }
+
+        stage('Create VM') {
+            sh '''
+            bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+            '''
+            sh """
+            if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-create.log"
+            fi
+            if [ -s ".kitchen/logs/kitchen.log" ]; then
+                mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-create.log"
+            fi
+            """
+        }
+
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.vagrant.d/insecure_private_key; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
+                    fi
+                    """
+                }
+                stage('Run Tests') {
+                    sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+// vim: ft=groovy

.ci/kitchen-opensuse15-py2

@@ -0,0 +1,159 @@
+@Library('salt@1.1') _
+
+// Define the maximum time, in hours, that a test run should run for
+def testrun_timeout = 6
+// Now define a global pipeline timeout. This is the test run timeout with one(1) additional
+// hour to allow for artifacts to be downloaded, if possible.
+def global_timeout = testrun_timeout + 1;
+
+def distro_name = 'opensuse'
+def distro_version = '15'
+def python_version = 'py2'
+def test_transport = 'ZeroMQ'
+def salt_target_branch = 'master'
+def golden_images_branch = '2019.2'
+def nox_passthrough_opts = '--ssh-tests'
+
+properties([
+    buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')),
+    parameters([
+        booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull')
+    ])
+])
+
+// Be sure to cancel any previously running builds
+def buildNumber = env.BUILD_NUMBER as int
+if (buildNumber > 1) {
+    // This will cancel the previous build which also defined a matching milestone
+    milestone(buildNumber - 1)
+}
+// Define a milestone for this build so that, if another build starts, this one will be aborted
+milestone(buildNumber)
+
+
+wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
+    withEnv([
+        'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml',
+        'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml',
+        'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml',
+        "NOX_ENV_NAME=runtests-${test_transport.toLowerCase()}",
+        'NOX_ENABLE_FROM_FILENAMES=true',
+        "NOX_PASSTHROUGH_OPTS=${nox_passthrough_opts}",
+        "SALT_TARGET_BRANCH=${salt_target_branch}",
+        "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
+        "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
+        'RBENV_VERSION=2.6.3',
+        "TEST_SUITE=${python_version}",
+        "TEST_PLATFORM=${distro_name}-${distro_version}",
+        "TEST_TRANSPORT=${test_transport}",
+        "FORCE_FULL=${params.runFull}",
+    ]) {
+        // Checkout the repo
+        stage('Clone') {
+            cleanWs notFailBuild: true
+            checkout scm
+            sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}'
+        }
+
+        // Setup the kitchen required bundle
+        stage('Setup') {
+            sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant'
+        }
+
+        stage('Create VM') {
+            retry(3) {
+                sh '''
+                t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
+                '''
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-create.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-create.log"
+                fi
+                """
+            }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
+        }
+
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
+                    fi
+                    """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
+                    }
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
+                        sh '''
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
+                        '''
+                    }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+// vim: ft=groovy

.ci/kitchen-opensuse15-py3

@@ -0,0 +1,159 @@
+@Library('salt@1.1') _
+
+// Define the maximum time, in hours, that a test run should run for
+def testrun_timeout = 6
+// Now define a global pipeline timeout. This is the test run timeout with one(1) additional
+// hour to allow for artifacts to be downloaded, if possible.
+def global_timeout = testrun_timeout + 1;
+
+def distro_name = 'opensuse'
+def distro_version = '15'
+def python_version = 'py3'
+def test_transport = 'ZeroMQ'
+def salt_target_branch = 'master'
+def golden_images_branch = '2019.2'
+def nox_passthrough_opts = '--ssh-tests'
+
+properties([
+    buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')),
+    parameters([
+        booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull')
+    ])
+])
+
+// Be sure to cancel any previously running builds
+def buildNumber = env.BUILD_NUMBER as int
+if (buildNumber > 1) {
+    // This will cancel the previous build which also defined a matching milestone
+    milestone(buildNumber - 1)
+}
+// Define a milestone for this build so that, if another build starts, this one will be aborted
+milestone(buildNumber)
+
+
+wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
+    withEnv([
+        'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml',
+        'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml',
+        'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml',
+        "NOX_ENV_NAME=runtests-${test_transport.toLowerCase()}",
+        'NOX_ENABLE_FROM_FILENAMES=true',
+        "NOX_PASSTHROUGH_OPTS=${nox_passthrough_opts}",
+        "SALT_TARGET_BRANCH=${salt_target_branch}",
+        "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
+        "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
+        'RBENV_VERSION=2.6.3',
+        "TEST_SUITE=${python_version}",
+        "TEST_PLATFORM=${distro_name}-${distro_version}",
+        "TEST_TRANSPORT=${test_transport}",
+        "FORCE_FULL=${params.runFull}",
+    ]) {
+        // Checkout the repo
+        stage('Clone') {
+            cleanWs notFailBuild: true
+            checkout scm
+            sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}'
+        }
+
+        // Setup the kitchen required bundle
+        stage('Setup') {
+            sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant'
+        }
+
+        stage('Create VM') {
+            retry(3) {
+                sh '''
+                t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
+                '''
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-create.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-create.log"
+                fi
+                """
+            }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
+        }
+
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
+                    fi
+                    """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
+                    }
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
+                        sh '''
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
+                        '''
+                    }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+// vim: ft=groovy

.ci/kitchen-ubuntu1604-py2

@@ -10,7 +10,7 @@ def distro_name = 'ubuntu'
 def distro_version = '1604'
 def python_version = 'py2'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-ubuntu1604-py2-tcp

@@ -10,7 +10,7 @@ def distro_name = 'ubuntu'
 def distro_version = '1604'
 def python_version = 'py2'
 def test_transport = 'TCP'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-ubuntu1604-py3

@@ -10,7 +10,7 @@ def distro_name = 'ubuntu'
 def distro_version = '1604'
 def python_version = 'py3'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-ubuntu1604-py3-tcp

@@ -10,7 +10,7 @@ def distro_name = 'ubuntu'
 def distro_version = '1604'
 def python_version = 'py3'
 def test_transport = 'TCP'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-ubuntu1804-py2

@@ -10,7 +10,7 @@ def distro_name = 'ubuntu'
 def distro_version = '1804'
 def python_version = 'py2'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-ubuntu1804-py3

@@ -10,7 +10,7 @@ def distro_name = 'ubuntu'
 def distro_version = '1804'
 def python_version = 'py3'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--ssh-tests'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -65,7 +65,8 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
             retry(3) {
                 sh '''
                 t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
-                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                cp -f ~/workspace/spot.yml .kitchen.local.yml
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM || (bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; rm .kitchen.local.yml; bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM); echo "ExitCode: $?";
                 '''
                 sh """
                 if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
@@ -76,74 +77,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout * 60 - 15, unit: 'MINUTES') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-windows2016-py2

@@ -10,7 +10,7 @@ def distro_name = 'windows'
 def distro_version = '2016'
 def python_version = 'py2'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--unit'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -76,74 +76,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout, unit: 'HOURS') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-windows2016-py3

@@ -10,7 +10,7 @@ def distro_name = 'windows'
 def distro_version = '2016'
 def python_version = 'py3'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
 def nox_passthrough_opts = '--unit'
 
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -76,74 +76,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout, unit: 'HOURS') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/kitchen-windows2019-py2

@@ -0,0 +1,158 @@
+@Library('salt@1.1') _
+
+// Define the maximum time, in hours, that a test run should run for
+def testrun_timeout = 8
+// Now define a global pipeline timeout. This is the test run timeout with one(1) additional
+// hour to allow for artifacts to be downloaded, if possible.
+def global_timeout = testrun_timeout + 1;
+
+def distro_name = 'windows'
+def distro_version = '2019'
+def python_version = 'py2'
+def test_transport = 'ZeroMQ'
+def salt_target_branch = 'master'
+def golden_images_branch = '2019.2'
+def nox_passthrough_opts = '--unit'
+
+properties([
+    buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')),
+    parameters([
+        booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull')
+    ])
+])
+
+// Be sure to cancel any previously running builds
+def buildNumber = env.BUILD_NUMBER as int
+if (buildNumber > 1) {
+    // This will cancel the previous build which also defined a matching milestone
+    milestone(buildNumber - 1)
+}
+// Define a milestone for this build so that, if another build starts, this one will be aborted
+milestone(buildNumber)
+
+
+wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
+    withEnv([
+        'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml',
+        'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml',
+        'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml',
+        "NOX_ENV_NAME=runtests-${test_transport.toLowerCase()}",
+        'NOX_ENABLE_FROM_FILENAMES=true',
+        "NOX_PASSTHROUGH_OPTS=${nox_passthrough_opts}",
+        "SALT_TARGET_BRANCH=${salt_target_branch}",
+        "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
+        "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
+        'RBENV_VERSION=2.6.3',
+        "TEST_SUITE=${python_version}",
+        "TEST_PLATFORM=${distro_name}-${distro_version}",
+        "TEST_TRANSPORT=${test_transport}",
+        "FORCE_FULL=${params.runFull}",
+    ]) {
+        // Checkout the repo
+        stage('Clone') {
+            cleanWs notFailBuild: true
+            checkout scm
+            sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}'
+        }
+
+        // Setup the kitchen required bundle
+        stage('Setup') {
+            sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant'
+        }
+
+        stage('Create VM') {
+            retry(3) {
+                sh '''
+                t=$(shuf -i 30-120 -n 1); echo "Sleeping $t seconds"; sleep $t
+                bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                '''
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-create.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-create.log"
+                fi
+                """
+            }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
+        }
+
+        try {
+            timeout(time: testrun_timeout, unit: 'HOURS') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
+                    fi
+                    """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
+                    }
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
+                        sh '''
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
+                        '''
+                    }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+// vim: ft=groovy

.ci/kitchen-windows2019-py3

@@ -1,18 +1,18 @@
 @Library('salt@1.1') _
 
 // Define the maximum time, in hours, that a test run should run for
-def testrun_timeout = 6
+def testrun_timeout = 8
 // Now define a global pipeline timeout. This is the test run timeout with one(1) additional
 // hour to allow for artifacts to be downloaded, if possible.
 def global_timeout = testrun_timeout + 1;
 
-def distro_name = 'debian'
-def distro_version = '8'
+def distro_name = 'windows'
+def distro_version = '2019'
 def python_version = 'py3'
 def test_transport = 'ZeroMQ'
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 def golden_images_branch = '2019.2'
-def nox_passthrough_opts = '--ssh-tests'
+def nox_passthrough_opts = '--unit'
 
 properties([
     buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')),
@@ -42,7 +42,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
         "SALT_TARGET_BRANCH=${salt_target_branch}",
         "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}",
         "CODECOV_FLAGS=${distro_name}${distro_version},${python_version},${test_transport.toLowerCase()}",
-        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin',
+        'PATH=~/.rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin',
         'RBENV_VERSION=2.6.3',
         "TEST_SUITE=${python_version}",
         "TEST_PLATFORM=${distro_name}-${distro_version}",
@@ -76,74 +76,77 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
                 fi
                 """
             }
+            sh '''
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'image_id:'
+            bundle exec kitchen diagnose $TEST_SUITE-$TEST_PLATFORM | grep 'instance_type:' -A5
+            '''
         }
 
-        sshagent(credentials: ['jenkins-testing-ssh-key']) {
-            sh 'ssh-add ~/.ssh/kitchen.pem'
-            try {
-                timeout(time: testrun_timeout, unit: 'HOURS') {
-                    stage('Converge VM') {
-                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
-                        fi
-                        """
-                    }
-                    stage('Run Tests') {
-                        withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
-                            sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
-                        }
-                    }
-                }
-            } finally {
-                try {
+        try {
+            timeout(time: testrun_timeout, unit: 'HOURS') {
+                stage('Converge VM') {
+                    sh '''
+                    ssh-agent /bin/bash -c 'ssh-add ~/.ssh/kitchen.pem; bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?"'
+                    '''
                     sh """
                     if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-converge.log"
                     fi
                     if [ -s ".kitchen/logs/kitchen.log" ]; then
-                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-converge.log"
                     fi
                     """
-                    stage('Download Artefacts') {
-                        withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
-                            sh '''
-                            bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
-                            '''
-                        }
-                        sh """
-                        if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
-                            mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
-                        fi
-                        if [ -s ".kitchen/logs/kitchen.log" ]; then
-                            mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
-                        fi
-                        """
+                }
+                stage('Run Tests') {
+                    withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) {
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";'
                     }
-                    archiveArtifacts(
-                        artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
-                        allowEmptyArchive: true
-                    )
-                    junit 'artifacts/xml-unittests-output/*.xml'
-                } finally {
-                    stage('Cleanup') {
+                }
+            }
+        } finally {
+            try {
+                sh """
+                if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                    mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-verify.log"
+                fi
+                if [ -s ".kitchen/logs/kitchen.log" ]; then
+                    mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-verify.log"
+                fi
+                """
+                stage('Download Artefacts') {
+                    withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){
                         sh '''
-                        bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                        bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0
                         '''
                     }
-                    stage('Upload Coverage') {
-                        script {
-                            withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
-                              sh '''
-                              if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
-                                  curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
-                              fi
-                              '''
-                            }
+                    sh """
+                    if [ -s ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ]; then
+                        mv ".kitchen/logs/${python_version}-${distro_name}-${distro_version}.log" ".kitchen/logs/${python_version}-${distro_name}-${distro_version}-download.log"
+                    fi
+                    if [ -s ".kitchen/logs/kitchen.log" ]; then
+                        mv ".kitchen/logs/kitchen.log" ".kitchen/logs/kitchen-download.log"
+                    fi
+                    """
+                }
+                archiveArtifacts(
+                    artifacts: "artifacts/*,artifacts/**/*,.kitchen/logs/*-create.log,.kitchen/logs/*-converge.log,.kitchen/logs/*-verify.log,.kitchen/logs/*-download.log,artifacts/xml-unittests-output/*.xml",
+                    allowEmptyArchive: true
+                )
+                junit 'artifacts/xml-unittests-output/*.xml'
+            } finally {
+                stage('Cleanup') {
+                    sh '''
+                    bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?";
+                    '''
+                }
+                stage('Upload Coverage') {
+                    script {
+                        withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
+                          sh '''
+                          if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
+                              (curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}") || true
+                          fi
+                          '''
                         }
                     }
                 }

.ci/lint

@@ -2,7 +2,7 @@
 
 // Define the maximum time, in hours, that a test run should run for
 def global_timeout = 3
-def salt_target_branch = '2019.2.1'
+def salt_target_branch = 'master'
 
 properties([
     buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')),

Gemfile

@@ -4,7 +4,7 @@ source 'https://rubygems.org'
 
 # Point this back at the test-kitchen package after >1.2.5 is relased
 gem 'test-kitchen', :git => 'https://github.com/dwoz/test-kitchen.git', :branch => 'keepalive_maxcount'
-gem 'kitchen-salt', :git => 'https://github.com/s0undt3ch/kitchen-salt.git', :branch => 'features/nox'
+gem 'kitchen-salt', :git => 'https://github.com/saltstack/kitchen-salt.git'
 gem 'kitchen-sync'
 gem 'git'
 

doc/man/salt-cloud.1

@@ -205,21 +205,21 @@ the provider\(aqs associated profiles, such as \fBdigitalocean\fP, or pass in
 .TP
 .B \-\-list\-locations=LIST_LOCATIONS
 Display a list of locations available in configured cloud providers. Pass
-the cloud provider that available locations are desired on, aka "linode",
+the cloud provider that available locations are desired on, such as "linode",
 or pass "all" to list locations for all configured cloud providers
 .UNINDENT
 .INDENT 0.0
 .TP
 .B \-\-list\-images=LIST_IMAGES
 Display a list of images available in configured cloud providers. Pass the
-cloud provider that available images are desired on, aka "linode", or pass
+cloud provider that available images are desired on, such as "linode", or pass
 "all" to list images for all configured cloud providers
 .UNINDENT
 .INDENT 0.0
 .TP
 .B \-\-list\-sizes=LIST_SIZES
 Display a list of sizes available in configured cloud providers. Pass the
-cloud provider that available sizes are desired on, aka "AWS", or pass
+cloud provider that available sizes are desired on, such as "AWS", or pass
 "all" to list sizes for all configured cloud providers
 .UNINDENT
 .SS Cloud Credentials

doc/man/salt.7

@@ -111754,7 +111754,7 @@ salt \(aq*\(aq archive.cmd_unzip template=jinja /tmp/zipfile.zip \(aq/tmp/{{grai
 .TP
 .B options
 Optional when using \fBzip\fP archives, ignored when usign other archives
-files. This is mostly used to overwrite exsiting files with \fBo\fP\&.
+files. This is mostly used to overwrite existing files with \fBo\fP\&.
 This options are only used when \fBunzip\fP binary is used.
 .sp
 New in version 2016.3.1.
@@ -342365,7 +342365,7 @@ a string of additional arguments to pass to the tar/zip command.
 .sp
 If this argument is not used, then the minion will attempt to use
 Python\(aqs native \fI\%tarfile\fP/\fI\%zipfile\fP support to extract it. For zip
-archives, this argument is mostly used to overwrite exsiting files with
+archives, this argument is mostly used to overwrite existing files with
 \fBo\fP\&.
 .sp
 Using this argument means that the \fBtar\fP or \fBunzip\fP command will be
@@ -345502,7 +345502,7 @@ The name of the virtual which will be deleted
 .INDENT 0.0
 .TP
 .B salt.states.bigip.list_monitor(hostname, username, password, monitor_type, name)
-A function to list an exsiting monitor.
+A function to list an existing monitor.
 .INDENT 7.0
 .TP
 .B hostname

doc/ref/cache/all/index.rst

@@ -1,7 +1,7 @@
 .. _all-salt.cache:
 
 =============
-Cache Modules
+cache modules
 =============
 
 .. currentmodule:: salt.cache

doc/ref/cli/salt-cloud.rst

@@ -146,19 +146,19 @@ Cloud Providers Listings
 .. option:: --list-locations=LIST_LOCATIONS
 
     Display a list of locations available in configured cloud providers. Pass
-    the cloud provider that available locations are desired on, aka "linode",
+    the cloud provider that available locations are desired on, such as "linode",
     or pass "all" to list locations for all configured cloud providers
 
 .. option:: --list-images=LIST_IMAGES
 
     Display a list of images available in configured cloud providers. Pass the
-    cloud provider that available images are desired on, aka "linode", or pass
+    cloud provider that available images are desired on, such as "linode", or pass
     "all" to list images for all configured cloud providers
 
 .. option:: --list-sizes=LIST_SIZES
 
     Display a list of sizes available in configured cloud providers. Pass the
-    cloud provider that available sizes are desired on, aka "AWS", or pass
+    cloud provider that available sizes are desired on, such as "AWS", or pass
     "all" to list sizes for all configured cloud providers
 
 Cloud Credentials

doc/topics/tutorials/multimaster.rst

@@ -95,7 +95,7 @@ redundant master. Both masters are first-class and have rights to the minions.
 .. note::
 
     Minions can automatically detect failed masters and attempt to reconnect
-    to reconnect to them quickly. To enable this functionality, set
+    to them quickly. To enable this functionality, set
     `master_alive_interval` in the minion config and specify a number of
     seconds to poll the masters for connection status.
 

requirements/static/debian-10.in

@@ -0,0 +1,44 @@
+# This is a compilation of requirements installed on salt-jenkins git.salt state run
+ansible; python_version < '3.0'
+apache-libcloud==2.0.0
+boto3
+boto>=2.46.0
+cffi
+cherrypy==17.3.0
+clustershell
+croniter>=0.3.0,!=0.3.22
+dnspython
+docker<4.0
+futures>=2.0; python_version < '3.0'
+GitPython
+jsonschema<=2.6.0
+junos-eznc
+jxmlease
+keyring==5.7.1
+kubernetes<4.0
+mock>=2.0.0; python_version < '3.6'
+more-itertools==5.0.0
+moto
+msgpack-python >= 0.4.2, != 0.5.5
+paramiko==2.2.3
+psutil
+# Let's install cryptodome instead of pycrypto because of pycrypto's outstanding security issues
+# PyCrypto, if pulled, will be removed from the generated static requirements
+pycryptodome
+pygit2
+pyinotify
+pylxd>=2.2.5
+pyopenssl
+python-etcd>0.4.2
+python-gnupg
+pyvmomi
+requests
+rfc3987
+salttesting==2017.6.1
+setproctitle
+strict_rfc3339
+supervisor==3.3.5; python_version < '3'
+timelib
+tornado<5.0
+virtualenv
+yamlordereddictloader

requirements/static/py3.4/zeromq-debian-10.txt

@@ -0,0 +1,120 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile -o requirements/static/py3.4/zeromq-debian-10.txt -v requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/debian-10.in
+#
+apache-libcloud==2.0.0
+asn1crypto==0.24.0        # via cryptography
+atomicwrites==1.3.0       # via pytest
+attrs==19.1.0             # via pytest
+aws-xray-sdk==0.95        # via moto
+backports.functools-lru-cache==1.5  # via cheroot
+backports.ssl-match-hostname==3.7.0.1  # via docker
+bcrypt==3.1.7             # via paramiko
+boto3==1.9.132
+boto==2.49.0
+botocore==1.12.132        # via boto3, moto, s3transfer
+cachetools==3.1.0         # via google-auth
+certifi==2019.3.9         # via kubernetes, requests
+cffi==1.12.2
+chardet==3.0.4            # via requests
+cheroot==6.5.4            # via cherrypy
+cherrypy==17.3.0
+clustershell==1.8.1
+contextlib2==0.5.5        # via cherrypy
+coverage==4.5.4           # via pytest-cov
+croniter==0.3.29
+cryptography==2.6.1       # via moto, paramiko, pylxd, pyopenssl
+dnspython==1.16.0
+docker-pycreds==0.4.0     # via docker
+docker==3.7.2
+docutils==0.14            # via botocore
+ecdsa==0.13.2             # via python-jose
+future==0.17.1            # via python-jose
+gitdb2==2.0.5             # via gitpython
+gitpython==2.1.11
+google-auth==1.6.3        # via kubernetes
+idna==2.8                 # via requests
+ipaddress==1.0.22         # via kubernetes
+jaraco.functools==2.0     # via tempora
+jinja2==2.10.1
+jmespath==0.9.4           # via boto3, botocore
+jsondiff==1.1.1           # via moto
+jsonpickle==1.1           # via aws-xray-sdk
+jsonschema==2.6.0
+junos-eznc==2.2.0
+jxmlease==1.0.1
+keyring==5.7.1
+kubernetes==3.0.0
+lxml==4.3.3               # via junos-eznc, ncclient
+markupsafe==1.1.1
+mock==2.0.0 ; python_version < "3.6"
+more-itertools==5.0.0
+moto==1.3.7
+msgpack-python==0.5.6
+msgpack==0.6.1
+ncclient==0.6.4           # via junos-eznc
+netaddr==0.7.19           # via junos-eznc
+paramiko==2.2.3
+pathlib2==2.3.3           # via pytest
+pbr==5.1.3                # via mock, pylxd
+pluggy==0.11.0            # via pytest
+portend==2.4              # via cherrypy
+psutil==5.6.1
+py==1.8.0                 # via pytest
+pyaml==19.4.1             # via moto
+pyasn1-modules==0.2.4     # via google-auth
+pyasn1==0.4.5             # via paramiko, pyasn1-modules, rsa
+pycparser==2.19           # via cffi
+# Next line explicitly commented out by pip-tools-compile because of the following regex: '^pycrypto==(.*)$'
+# pycrypto==2.6.1 ; sys_platform != "win32"
+pycryptodome==3.8.1
+pygit2==0.28.1
+pyinotify==0.9.6
+pylxd==2.2.9
+pynacl==1.3.0             # via paramiko
+pyopenssl==19.0.0
+pyserial==3.4             # via junos-eznc
+pytest-cov==2.7.1
+pytest-helpers-namespace==2019.1.8
+pytest-salt-runtests-bridge==2019.1.30
+pytest-salt==2018.12.8
+pytest-tempdir==2018.8.11
+pytest-timeout==1.3.3
+pytest==4.4.2
+python-dateutil==2.8.0    # via botocore, croniter, kubernetes, moto, pylxd
+python-etcd==0.4.5
+python-gnupg==0.4.4
+python-jose==2.0.2        # via moto
+pytz==2019.1              # via moto, tempora
+pyvmomi==6.7.1.2018.12
+pyyaml==3.13
+pyzmq==17.0.0 ; python_version == "3.4"
+requests-toolbelt==0.9.1  # via pylxd
+requests-unixsocket==0.1.5  # via pylxd
+requests==2.21.0
+responses==0.10.6         # via moto
+rfc3987==1.3.8
+rsa==4.0                  # via google-auth
+s3transfer==0.2.0         # via boto3
+salttesting==2017.6.1
+scandir==1.10.0           # via pathlib2
+scp==0.13.2               # via junos-eznc
+selectors2==2.0.1         # via ncclient
+setproctitle==1.1.10
+six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, pathlib2, pygit2, pylxd, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, websocket-client
+smmap2==2.0.5             # via gitdb2
+strict-rfc3339==0.7
+tempora==1.14.1           # via portend
+timelib==0.2.4
+tornado==4.5.3 ; python_version >= "3.4"
+urllib3==1.24.3           # via botocore, kubernetes, python-etcd, requests, requests-unixsocket
+virtualenv==16.4.3
+websocket-client==0.40.0  # via docker, kubernetes
+werkzeug==0.15.2          # via moto
+wrapt==1.11.1             # via aws-xray-sdk
+ws4py==0.5.1              # via pylxd
+xmltodict==0.12.0         # via moto
+yamlordereddictloader==0.4.0
+zc.lockfile==1.4          # via cherrypy

requirements/static/py3.5/zeromq-debian-10.txt

@@ -0,0 +1,117 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile -o requirements/static/py3.5/zeromq-debian-10.txt -v requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/debian-10.in
+#
+apache-libcloud==2.0.0
+asn1crypto==0.24.0        # via cryptography
+atomicwrites==1.3.0       # via pytest
+attrs==19.1.0             # via pytest
+aws-xray-sdk==0.95        # via moto
+backports.functools-lru-cache==1.5  # via cheroot
+bcrypt==3.1.7             # via paramiko
+boto3==1.9.132
+boto==2.49.0
+botocore==1.12.132        # via boto3, moto, s3transfer
+cachetools==3.1.0         # via google-auth
+certifi==2019.3.9         # via kubernetes, requests
+cffi==1.12.2
+chardet==3.0.4            # via requests
+cheroot==6.5.4            # via cherrypy
+cherrypy==17.3.0
+clustershell==1.8.1
+contextlib2==0.5.5        # via cherrypy
+coverage==4.5.4           # via pytest-cov
+croniter==0.3.29
+cryptography==2.6.1       # via moto, paramiko, pylxd, pyopenssl
+dnspython==1.16.0
+docker-pycreds==0.4.0     # via docker
+docker==3.7.2
+docutils==0.14            # via botocore
+ecdsa==0.13.2             # via python-jose
+future==0.17.1            # via python-jose
+gitdb2==2.0.5             # via gitpython
+gitpython==2.1.11
+google-auth==1.6.3        # via kubernetes
+idna==2.8                 # via requests
+ipaddress==1.0.22         # via kubernetes
+jaraco.functools==2.0     # via tempora
+jinja2==2.10.1
+jmespath==0.9.4           # via boto3, botocore
+jsondiff==1.1.1           # via moto
+jsonpickle==1.1           # via aws-xray-sdk
+jsonschema==2.6.0
+junos-eznc==2.2.0
+jxmlease==1.0.1
+keyring==5.7.1
+kubernetes==3.0.0
+lxml==4.3.3               # via junos-eznc, ncclient
+markupsafe==1.1.1
+mock==2.0.0 ; python_version < "3.6"
+more-itertools==5.0.0
+moto==1.3.7
+msgpack-python==0.5.6
+msgpack==0.6.1
+ncclient==0.6.4           # via junos-eznc
+netaddr==0.7.19           # via junos-eznc
+paramiko==2.2.3
+pathlib2==2.3.3           # via pytest
+pbr==5.1.3                # via mock, pylxd
+pluggy==0.11.0            # via pytest
+portend==2.4              # via cherrypy
+psutil==5.6.1
+py==1.8.0                 # via pytest
+pyaml==19.4.1             # via moto
+pyasn1-modules==0.2.4     # via google-auth
+pyasn1==0.4.5             # via paramiko, pyasn1-modules, rsa
+pycparser==2.19           # via cffi
+# Next line explicitly commented out by pip-tools-compile because of the following regex: '^pycrypto==(.*)$'
+# pycrypto==2.6.1 ; sys_platform != "win32"
+pycryptodome==3.8.1
+pygit2==0.28.1
+pyinotify==0.9.6
+pylxd==2.2.9
+pynacl==1.3.0             # via paramiko
+pyopenssl==19.0.0
+pyserial==3.4             # via junos-eznc
+pytest-cov==2.7.1
+pytest-helpers-namespace==2019.1.8
+pytest-salt-runtests-bridge==2019.1.30
+pytest-salt==2018.12.8
+pytest-tempdir==2018.8.11
+pytest-timeout==1.3.3
+pytest==4.4.2
+python-dateutil==2.8.0    # via botocore, croniter, kubernetes, moto, pylxd
+python-etcd==0.4.5
+python-gnupg==0.4.4
+python-jose==2.0.2        # via moto
+pytz==2019.1              # via moto, tempora
+pyvmomi==6.7.1.2018.12
+pyyaml==3.13
+pyzmq==18.0.1 ; python_version != "3.4"
+requests-toolbelt==0.9.1  # via pylxd
+requests-unixsocket==0.1.5  # via pylxd
+requests==2.21.0
+responses==0.10.6         # via moto
+rfc3987==1.3.8
+rsa==4.0                  # via google-auth
+s3transfer==0.2.0         # via boto3
+salttesting==2017.6.1
+scp==0.13.2               # via junos-eznc
+setproctitle==1.1.10
+six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, pathlib2, pygit2, pylxd, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, websocket-client
+smmap2==2.0.5             # via gitdb2
+strict-rfc3339==0.7
+tempora==1.14.1           # via portend
+timelib==0.2.4
+tornado==4.5.3 ; python_version >= "3.4"
+urllib3==1.24.3           # via botocore, kubernetes, python-etcd, requests, requests-unixsocket
+virtualenv==16.4.3
+websocket-client==0.40.0  # via docker, kubernetes
+werkzeug==0.15.2          # via moto
+wrapt==1.11.1             # via aws-xray-sdk
+ws4py==0.5.1              # via pylxd
+xmltodict==0.12.0         # via moto
+yamlordereddictloader==0.4.0
+zc.lockfile==1.4          # via cherrypy

requirements/static/py3.6/zeromq-debian-10.txt

@@ -0,0 +1,116 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile -o requirements/static/py3.6/zeromq-debian-10.txt -v requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/debian-10.in
+#
+apache-libcloud==2.0.0
+asn1crypto==0.24.0        # via cryptography
+atomicwrites==1.3.0       # via pytest
+attrs==19.1.0             # via pytest
+aws-xray-sdk==0.95        # via moto
+backports.functools-lru-cache==1.5  # via cheroot
+bcrypt==3.1.7             # via paramiko
+boto3==1.9.132
+boto==2.49.0
+botocore==1.12.132        # via boto3, moto, s3transfer
+cachetools==3.1.0         # via google-auth
+certifi==2019.3.9         # via kubernetes, requests
+cffi==1.12.2
+chardet==3.0.4            # via requests
+cheroot==6.5.4            # via cherrypy
+cherrypy==17.3.0
+clustershell==1.8.1
+contextlib2==0.5.5        # via cherrypy
+coverage==4.5.4           # via pytest-cov
+croniter==0.3.29
+cryptography==2.6.1       # via moto, paramiko, pylxd, pyopenssl
+dnspython==1.16.0
+docker-pycreds==0.4.0     # via docker
+docker==3.7.2
+docutils==0.14            # via botocore
+ecdsa==0.13.2             # via python-jose
+future==0.17.1            # via python-jose
+gitdb2==2.0.5             # via gitpython
+gitpython==2.1.11
+google-auth==1.6.3        # via kubernetes
+idna==2.8                 # via requests
+ipaddress==1.0.22         # via kubernetes
+jaraco.functools==2.0     # via tempora
+jinja2==2.10.1
+jmespath==0.9.4           # via boto3, botocore
+jsondiff==1.1.1           # via moto
+jsonpickle==1.1           # via aws-xray-sdk
+jsonschema==2.6.0
+junos-eznc==2.2.0
+jxmlease==1.0.1
+keyring==5.7.1
+kubernetes==3.0.0
+lxml==4.3.3               # via junos-eznc, ncclient
+markupsafe==1.1.1
+mock==2.0.0               # via moto
+more-itertools==5.0.0
+moto==1.3.7
+msgpack-python==0.5.6
+msgpack==0.6.1
+ncclient==0.6.4           # via junos-eznc
+netaddr==0.7.19           # via junos-eznc
+paramiko==2.2.3
+pbr==5.1.3                # via mock, pylxd
+pluggy==0.11.0            # via pytest
+portend==2.4              # via cherrypy
+psutil==5.6.1
+py==1.8.0                 # via pytest
+pyaml==19.4.1             # via moto
+pyasn1-modules==0.2.4     # via google-auth
+pyasn1==0.4.5             # via paramiko, pyasn1-modules, rsa
+pycparser==2.19           # via cffi
+# Next line explicitly commented out by pip-tools-compile because of the following regex: '^pycrypto==(.*)$'
+# pycrypto==2.6.1 ; sys_platform != "win32"
+pycryptodome==3.8.1
+pygit2==0.28.1
+pyinotify==0.9.6
+pylxd==2.2.9
+pynacl==1.3.0             # via paramiko
+pyopenssl==19.0.0
+pyserial==3.4             # via junos-eznc
+pytest-cov==2.7.1
+pytest-helpers-namespace==2019.1.8
+pytest-salt-runtests-bridge==2019.1.30
+pytest-salt==2018.12.8
+pytest-tempdir==2018.8.11
+pytest-timeout==1.3.3
+pytest==4.4.2
+python-dateutil==2.8.0    # via botocore, croniter, kubernetes, moto, pylxd
+python-etcd==0.4.5
+python-gnupg==0.4.4
+python-jose==2.0.2        # via moto
+pytz==2019.1              # via moto, tempora
+pyvmomi==6.7.1.2018.12
+pyyaml==3.13
+pyzmq==18.0.1 ; python_version != "3.4"
+requests-toolbelt==0.9.1  # via pylxd
+requests-unixsocket==0.1.5  # via pylxd
+requests==2.21.0
+responses==0.10.6         # via moto
+rfc3987==1.3.8
+rsa==4.0                  # via google-auth
+s3transfer==0.2.0         # via boto3
+salttesting==2017.6.1
+scp==0.13.2               # via junos-eznc
+setproctitle==1.1.10
+six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, pygit2, pylxd, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, websocket-client
+smmap2==2.0.5             # via gitdb2
+strict-rfc3339==0.7
+tempora==1.14.1           # via portend
+timelib==0.2.4
+tornado==4.5.3 ; python_version >= "3.4"
+urllib3==1.24.3           # via botocore, kubernetes, python-etcd, requests, requests-unixsocket
+virtualenv==16.4.3
+websocket-client==0.40.0  # via docker, kubernetes
+werkzeug==0.15.2          # via moto
+wrapt==1.11.1             # via aws-xray-sdk
+ws4py==0.5.1              # via pylxd
+xmltodict==0.12.0         # via moto
+yamlordereddictloader==0.4.0
+zc.lockfile==1.4          # via cherrypy

requirements/static/py3.7/zeromq-debian-10.txt

@@ -0,0 +1,116 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile -o requirements/static/py3.7/zeromq-debian-10.txt -v requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/debian-10.in
+#
+apache-libcloud==2.0.0
+asn1crypto==0.24.0        # via cryptography
+atomicwrites==1.3.0       # via pytest
+attrs==19.1.0             # via pytest
+aws-xray-sdk==0.95        # via moto
+backports.functools-lru-cache==1.5  # via cheroot
+bcrypt==3.1.7             # via paramiko
+boto3==1.9.132
+boto==2.49.0
+botocore==1.12.132        # via boto3, moto, s3transfer
+cachetools==3.1.0         # via google-auth
+certifi==2019.3.9         # via kubernetes, requests
+cffi==1.12.2
+chardet==3.0.4            # via requests
+cheroot==6.5.4            # via cherrypy
+cherrypy==17.3.0
+clustershell==1.8.1
+contextlib2==0.5.5        # via cherrypy
+coverage==4.5.4           # via pytest-cov
+croniter==0.3.29
+cryptography==2.6.1       # via moto, paramiko, pylxd, pyopenssl
+dnspython==1.16.0
+docker-pycreds==0.4.0     # via docker
+docker==3.7.2
+docutils==0.14            # via botocore
+ecdsa==0.13.2             # via python-jose
+future==0.17.1            # via python-jose
+gitdb2==2.0.5             # via gitpython
+gitpython==2.1.11
+google-auth==1.6.3        # via kubernetes
+idna==2.8                 # via requests
+ipaddress==1.0.22         # via kubernetes
+jaraco.functools==2.0     # via tempora
+jinja2==2.10.1
+jmespath==0.9.4           # via boto3, botocore
+jsondiff==1.1.1           # via moto
+jsonpickle==1.1           # via aws-xray-sdk
+jsonschema==2.6.0
+junos-eznc==2.2.0
+jxmlease==1.0.1
+keyring==5.7.1
+kubernetes==3.0.0
+lxml==4.3.3               # via junos-eznc, ncclient
+markupsafe==1.1.1
+mock==2.0.0               # via moto
+more-itertools==5.0.0
+moto==1.3.7
+msgpack-python==0.5.6
+msgpack==0.6.1
+ncclient==0.6.4           # via junos-eznc
+netaddr==0.7.19           # via junos-eznc
+paramiko==2.2.3
+pbr==5.1.3                # via mock, pylxd
+pluggy==0.11.0            # via pytest
+portend==2.4              # via cherrypy
+psutil==5.6.1
+py==1.8.0                 # via pytest
+pyaml==19.4.1             # via moto
+pyasn1-modules==0.2.4     # via google-auth
+pyasn1==0.4.5             # via paramiko, pyasn1-modules, rsa
+pycparser==2.19           # via cffi
+# Next line explicitly commented out by pip-tools-compile because of the following regex: '^pycrypto==(.*)$'
+# pycrypto==2.6.1 ; sys_platform != "win32"
+pycryptodome==3.8.1
+pygit2==0.28.1
+pyinotify==0.9.6
+pylxd==2.2.9
+pynacl==1.3.0             # via paramiko
+pyopenssl==19.0.0
+pyserial==3.4             # via junos-eznc
+pytest-cov==2.7.1
+pytest-helpers-namespace==2019.1.8
+pytest-salt-runtests-bridge==2019.1.30
+pytest-salt==2018.12.8
+pytest-tempdir==2018.8.11
+pytest-timeout==1.3.3
+pytest==4.4.2
+python-dateutil==2.8.0    # via botocore, croniter, kubernetes, moto, pylxd
+python-etcd==0.4.5
+python-gnupg==0.4.4
+python-jose==2.0.2        # via moto
+pytz==2019.1              # via moto, tempora
+pyvmomi==6.7.1.2018.12
+pyyaml==3.13
+pyzmq==18.0.1 ; python_version != "3.4"
+requests-toolbelt==0.9.1  # via pylxd
+requests-unixsocket==0.1.5  # via pylxd
+requests==2.21.0
+responses==0.10.6         # via moto
+rfc3987==1.3.8
+rsa==4.0                  # via google-auth
+s3transfer==0.2.0         # via boto3
+salttesting==2017.6.1
+scp==0.13.2               # via junos-eznc
+setproctitle==1.1.10
+six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, pygit2, pylxd, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, websocket-client
+smmap2==2.0.5             # via gitdb2
+strict-rfc3339==0.7
+tempora==1.14.1           # via portend
+timelib==0.2.4
+tornado==4.5.3 ; python_version >= "3.4"
+urllib3==1.24.3           # via botocore, kubernetes, python-etcd, requests, requests-unixsocket
+virtualenv==16.4.3
+websocket-client==0.40.0  # via docker, kubernetes
+werkzeug==0.15.2          # via moto
+wrapt==1.11.1             # via aws-xray-sdk
+ws4py==0.5.1              # via pylxd
+xmltodict==0.12.0         # via moto
+yamlordereddictloader==0.4.0
+zc.lockfile==1.4          # via cherrypy

salt/cloud/clouds/saltify.py

@@ -300,7 +300,7 @@ def get_configured_provider():
 
 def _verify(vm_):
     '''
-    Verify credentials for an exsiting system
+    Verify credentials for an existing system
     '''
     log.info('Verifying credentials for %s', vm_['name'])
 

salt/modules/archive.py

@@ -889,7 +889,7 @@ def cmd_unzip(zip_file,
 
     options
         Optional when using ``zip`` archives, ignored when usign other archives
-        files. This is mostly used to overwrite exsiting files with ``o``.
+        files. This is mostly used to overwrite existing files with ``o``.
         This options are only used when ``unzip`` binary is used.
 
         .. versionadded:: 2016.3.1

salt/modules/pillar.py

@@ -614,7 +614,7 @@ def filter_by(lookup_dict,
         on that particular OS.
 
         The dictionary key can be a globbing pattern. The function will return
-        the corresponding ``lookup_dict`` value where the pilalr value matches
+        the corresponding ``lookup_dict`` value where the pillar value matches
         the  pattern. For example:
 
         .. code-block:: bash

salt/modules/state.py

@@ -683,10 +683,14 @@ def apply_(mods=None, **kwargs):
 
         # Run the states configured in salt://stuff.sls (or salt://stuff/init.sls)
         salt '*' state.apply stuff
+
         # Run the states configured in salt://stuff.sls (or salt://stuff/init.sls)
         # and salt://pkgs.sls (or salt://pkgs/init.sls).
         salt '*' state.apply stuff,pkgs
 
+        # Run the states configured in a more deeply nested directory such as salt://my/organized/stuff.sls (or salt://my/organized/stuff/init.sls)
+        salt '*' state.apply my.organized.stuff
+
     The following additional arguments are also accepted when applying
     individual SLS files:
 
@@ -1176,14 +1180,13 @@ def sls(mods, test=None, exclude=None, queue=False, sync_mods=None, **kwargs):
         used, all Pillar environments will be merged together.
 
     localconfig
-
         Optionally, instead of using the minion config, load minion opts from
         the file specified by this argument, and then merge them with the
         options from the minion config. This functionality allows for specific
         states to be run with their own custom minion configuration, including
         different pillars, file_roots, etc.
 
-    mock:
+    mock
         The mock option allows for the state run to execute without actually
         calling any states. This then returns a mocked return which will show
         the requisite ordering as well as fully validate the state run.
@@ -1205,9 +1208,17 @@ def sls(mods, test=None, exclude=None, queue=False, sync_mods=None, **kwargs):
 
     .. code-block:: bash
 
-        salt '*' state.sls core,edit.vim dev
-        salt '*' state.sls core exclude="[{'id': 'id_to_exclude'}, {'sls': 'sls_to_exclude'}]"
+        # Run the states configured in salt://example.sls (or salt://example/init.sls)
+        salt '*' state.apply example
 
+        # Run the states configured in salt://core.sls (or salt://core/init.sls)
+        # and salt://edit/vim.sls (or salt://edit/vim/init.sls)
+        salt '*' state.sls core,edit.vim
+
+        # Run the states configured in a more deeply nested directory such as salt://my/nested/state.sls (or salt://my/nested/state/init.sls)
+        salt '*' state.sls my.nested.state
+
+        salt '*' state.sls core exclude="[{'id': 'id_to_exclude'}, {'sls': 'sls_to_exclude'}]"
         salt '*' state.sls myslsfile pillar="{foo: 'Foo!', bar: 'Bar!'}"
     '''
     concurrent = kwargs.get('concurrent', False)

salt/states/archive.py

@@ -453,7 +453,7 @@ def extracted(name,
 
         If this argument is not used, then the minion will attempt to use
         Python's native tarfile_/zipfile_ support to extract it. For zip
-        archives, this argument is mostly used to overwrite exsiting files with
+        archives, this argument is mostly used to overwrite existing files with
         ``o``.
 
         Using this argument means that the ``tar`` or ``unzip`` command will be

salt/states/bigip.py

@@ -2491,7 +2491,7 @@ def delete_virtual(hostname, username, password, name):
 
 def list_monitor(hostname, username, password, monitor_type, name):
     '''
-    A function to list an exsiting monitor.
+    A function to list an existing monitor.
 
     hostname
         The host/address of the bigip device

salt/states/file.py

@@ -2228,7 +2228,7 @@ def managed(name,
         .. versionadded:: 0.17.0
         .. versionchanged:: 2016.11.0
             contents_pillar can also be a list, and the pillars will be
-            concatinated together to form one file.
+            concatenated together to form one file.
 
 
         Operates like ``contents``, but draws from a value stored in pillar,
@@ -2269,7 +2269,7 @@ def managed(name,
         .. note::
             The private key above is shortened to keep the example brief, but
             shows how to do multiline string in YAML. The key is followed by a
-            pipe character, and the mutliline string is indented two more
+            pipe character, and the multiline string is indented two more
             spaces.
 
             To avoid the hassle of creating an indented multiline YAML string,
@@ -3176,7 +3176,7 @@ def directory(name,
         example: ``{'Administrators': {'perms': 'full_control', 'applies_to':
         'this_folder_only'}}`` Can be a single basic perm or a list of advanced
         perms. ``perms`` must be specified. ``applies_to`` is optional and
-        defaults to ``this_folder_subfoler_files``.
+        defaults to ``this_folder_subfolder_files``.
 
         .. versionadded:: 2017.7.0
 

salt/utils/parsers.py

@@ -1664,7 +1664,7 @@ class CloudProvidersListsMixIn(six.with_metaclass(MixInMeta, object)):
             default=None,
             help=('Display a list of locations available in configured cloud '
                   'providers. Pass the cloud provider that available '
-                  'locations are desired on, aka "linode", or pass "all" to '
+                  'locations are desired on, such as "linode", or pass "all" to '
                   'list locations for all configured cloud providers.')
         )
         group.add_option(
@@ -1672,7 +1672,7 @@ class CloudProvidersListsMixIn(six.with_metaclass(MixInMeta, object)):
             default=None,
             help=('Display a list of images available in configured cloud '
                   'providers. Pass the cloud provider that available images '
-                  'are desired on, aka "linode", or pass "all" to list images '
+                  'are desired on, such as "linode", or pass "all" to list images '
                   'for all configured cloud providers.')
         )
         group.add_option(
@@ -1680,7 +1680,7 @@ class CloudProvidersListsMixIn(six.with_metaclass(MixInMeta, object)):
             default=None,
             help=('Display a list of sizes available in configured cloud '
                   'providers. Pass the cloud provider that available sizes '
-                  'are desired on, aka "AWS", or pass "all" to list sizes '
+                  'are desired on, such as "AWS", or pass "all" to list sizes '
                   'for all configured cloud providers.')
         )
         self.add_option_group(group)

tests/integration/__init__.py

@@ -296,7 +296,7 @@ class TestDaemon(object):
                 daemon_class=SaltMaster,
                 bin_dir_path=SCRIPT_DIR,
                 fail_hard=True,
-                start_timeout=60)
+                start_timeout=120)
             sys.stdout.write(
                 '\r{0}\r'.format(
                     ' ' * getattr(self.parser.options, 'output_columns', PNUM)
@@ -333,7 +333,7 @@ class TestDaemon(object):
                 daemon_class=SaltMinion,
                 bin_dir_path=SCRIPT_DIR,
                 fail_hard=True,
-                start_timeout=60)
+                start_timeout=120)
             sys.stdout.write(
                 '\r{0}\r'.format(
                     ' ' * getattr(self.parser.options, 'output_columns', PNUM)
@@ -370,7 +370,7 @@ class TestDaemon(object):
                 daemon_class=SaltMinion,
                 bin_dir_path=SCRIPT_DIR,
                 fail_hard=True,
-                start_timeout=60)
+                start_timeout=120)
             sys.stdout.write(
                 '\r{0}\r'.format(
                     ' ' * getattr(self.parser.options, 'output_columns', PNUM)
@@ -408,7 +408,7 @@ class TestDaemon(object):
                 daemon_class=SaltMaster,
                 bin_dir_path=SCRIPT_DIR,
                 fail_hard=True,
-                start_timeout=60)
+                start_timeout=120)
             sys.stdout.write(
                 '\r{0}\r'.format(
                     ' ' * getattr(self.parser.options, 'output_columns', PNUM)
@@ -445,7 +445,7 @@ class TestDaemon(object):
                 daemon_class=SaltSyndic,
                 bin_dir_path=SCRIPT_DIR,
                 fail_hard=True,
-                start_timeout=60)
+                start_timeout=120)
             sys.stdout.write(
                 '\r{0}\r'.format(
                     ' ' * getattr(self.parser.options, 'output_columns', PNUM)
@@ -484,7 +484,7 @@ class TestDaemon(object):
                     daemon_class=SaltProxy,
                     bin_dir_path=SCRIPT_DIR,
                     fail_hard=True,
-                    start_timeout=60)
+                    start_timeout=120)
                 sys.stdout.write(
                     '\r{0}\r'.format(
                         ' ' * getattr(self.parser.options, 'output_columns', PNUM)

tests/integration/modules/test_pkg.py

@@ -10,35 +10,39 @@ from tests.support.case import ModuleCase
 from tests.support.mixins import SaltReturnAssertsMixin
 from tests.support.helpers import (
     destructiveTest,
-    flaky,
     requires_network,
     requires_salt_modules,
-)
-from tests.support.unit import skipIf
+    requires_system_grains)
 
 # Import Salt libs
+from salt.utils import six
 import salt.utils.pkg
 import salt.utils.platform
 
 
-@flaky
 class PkgModuleTest(ModuleCase, SaltReturnAssertsMixin):
     '''
     Validate the pkg module
     '''
-    def setUp(self):
+
+    @classmethod
+    @requires_system_grains
+    def setUpClass(cls, grains):
+        cls.ctx = {}
+        cls.pkg = 'figlet'
         if salt.utils.platform.is_windows():
+            cls.pkg = 'putty'
+        elif salt.utils.platform.is_darwin():
+            cls.pkg = 'wget' if int(grains['osmajorrelease']) >= 13 else 'htop'
+        elif grains['os_family'] == 'RedHat':
+            cls.pkg = 'units'
+
+    def setUp(self):
+        if 'refresh' not in self.ctx:
             self.run_function('pkg.refresh_db')
+            self.ctx['refresh'] = True
 
-        os_release = self.run_function('grains.get', ['osrelease'])
-
-        if salt.utils.platform.is_darwin() and int(os_release.split('.')[1]) >= 13:
-            self.pkg = 'wget'
-        elif salt.utils.platform.is_windows():
-            self.pkg = 'putty'
-        else:
-            self.pkg = 'htop'
-
+    @requires_salt_modules('pkg.list_pkgs')
     def test_list(self):
         '''
         verify that packages are installed
@@ -47,53 +51,50 @@ class PkgModuleTest(ModuleCase, SaltReturnAssertsMixin):
         self.assertNotEqual(len(ret.keys()), 0)
 
     @requires_salt_modules('pkg.version_cmp')
-    def test_version_cmp(self):
+    @requires_system_grains
+    def test_version_cmp(self, grains):
         '''
         test package version comparison on supported platforms
         '''
         func = 'pkg.version_cmp'
-        os_family = self.run_function('grains.item', ['os_family'])['os_family']
-        if os_family == 'Debian':
+        if grains['os_family'] == 'Debian':
             lt = ['0.2.4-0ubuntu1', '0.2.4.1-0ubuntu1']
             eq = ['0.2.4-0ubuntu1', '0.2.4-0ubuntu1']
             gt = ['0.2.4.1-0ubuntu1', '0.2.4-0ubuntu1']
-
-            self.assertEqual(self.run_function(func, lt), -1)
-            self.assertEqual(self.run_function(func, eq), 0)
-            self.assertEqual(self.run_function(func, gt), 1)
-        elif os_family == 'Suse':
+        elif grains['os_family'] == 'Suse':
             lt = ['2.3.0-1', '2.3.1-15.1']
             eq = ['2.3.1-15.1', '2.3.1-15.1']
             gt = ['2.3.2-15.1', '2.3.1-15.1']
-
-            self.assertEqual(self.run_function(func, lt), -1)
-            self.assertEqual(self.run_function(func, eq), 0)
-            self.assertEqual(self.run_function(func, gt), 1)
         else:
-            self.skipTest('{0} is unavailable on {1}'.format(func, os_family))
+            lt = ['2.3.0', '2.3.1']
+            eq = ['2.3.1', '2.3.1']
+            gt = ['2.3.2', '2.3.1']
+
+        self.assertEqual(self.run_function(func, lt), -1)
+        self.assertEqual(self.run_function(func, eq), 0)
+        self.assertEqual(self.run_function(func, gt), 1)
 
-    @requires_salt_modules('pkg.mod_repo', 'pkg.del_repo')
-    @requires_network()
     @destructiveTest
-    def test_mod_del_repo(self):
+    @requires_salt_modules('pkg.mod_repo', 'pkg.del_repo', 'pkg.get_repo')
+    @requires_network()
+    @requires_system_grains
+    def test_mod_del_repo(self, grains):
         '''
         test modifying and deleting a software repository
         '''
-        os_grain = self.run_function('grains.item', ['os'])['os']
         repo = None
 
         try:
-            if os_grain == 'Ubuntu':
+            if grains['os'] == 'Ubuntu':
                 repo = 'ppa:otto-kesselgulasch/gimp-edge'
                 uri = 'http://ppa.launchpad.net/otto-kesselgulasch/gimp-edge/ubuntu'
                 ret = self.run_function('pkg.mod_repo', [repo, 'comps=main'])
                 self.assertNotEqual(ret, {})
                 ret = self.run_function('pkg.get_repo', [repo])
 
-                if not isinstance(ret, dict):
-                    self.fail(
-                        'The \'pkg.get_repo\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
-                    )
+                self.assertIsInstance(ret, dict,
+                                      'The \'pkg.get_repo\' command did not return the excepted dictionary. '
+                                      'Output:\n{}'.format(ret))
 
                 self.assertEqual(
                     ret['uri'],
@@ -102,17 +103,12 @@ class PkgModuleTest(ModuleCase, SaltReturnAssertsMixin):
                         pprint.pformat(ret)
                     )
                 )
-            elif os_grain == 'CentOS':
-                major_release = int(
-                    self.run_function(
-                        'grains.item',
-                        ['osmajorrelease']
-                    )['osmajorrelease']
-                )
+            elif grains['os_family'] == 'RedHat':
                 repo = 'saltstack'
-                name = 'SaltStack repo for RHEL/CentOS {0}'.format(major_release)
-                baseurl = 'http://repo.saltstack.com/yum/redhat/{0}/x86_64/latest/'.format(major_release)
-                gpgkey = 'https://repo.saltstack.com/yum/rhel{0}/SALTSTACK-GPG-KEY.pub'.format(major_release)
+                name = 'SaltStack repo for RHEL/CentOS {0}'.format(grains['osmajorrelease'])
+                baseurl = 'http://repo.saltstack.com/yum/redhat/{0}/x86_64/latest/'.format(grains['osmajorrelease'])
+                gpgkey = 'https://repo.saltstack.com/yum/rhel{0}/SALTSTACK-GPG-KEY.pub'.format(
+                    grains['osmajorrelease'])
                 gpgcheck = 1
                 enabled = 1
                 ret = self.run_function(
@@ -142,17 +138,12 @@ class PkgModuleTest(ModuleCase, SaltReturnAssertsMixin):
         test finding the package owning a file
         '''
         func = 'pkg.owner'
-        available = self.run_function('sys.doc', [func])
+        ret = self.run_function(func, ['/bin/ls'])
+        self.assertNotEqual(len(ret), 0)
 
-        if available:
-            ret = self.run_function(func, ['/bin/ls'])
-            self.assertNotEqual(len(ret), 0)
-        else:
-            os_grain = self.run_function('grains.item', ['os'])['os']
-            self.skipTest('{0} is unavailable on {1}'.format(func, os_grain))
-
-    @requires_network()
     @destructiveTest
+    @requires_salt_modules('pkg.version', 'pkg.install', 'pkg.remove')
+    @requires_network()
     def test_install_remove(self):
         '''
         successfully install and uninstall a package
@@ -177,118 +168,108 @@ class PkgModuleTest(ModuleCase, SaltReturnAssertsMixin):
             test_install()
             test_remove()
 
-    @skipIf(True, 'WAR ROOM TEMPORARY SKIP')            # needs to be rewritten to allow for dnf on Fedora 30 and RHEL 8
-    @skipIf(salt.utils.platform.is_windows(), "Skip on windows")
-    @requires_salt_modules('pkg.hold', 'pkg.unhold')
-    @requires_network()
     @destructiveTest
-    def test_hold_unhold(self):
+    @requires_salt_modules('pkg.hold', 'pkg.unhold', 'pkg.install', 'pkg.version', 'pkg.remove')
+    @requires_network()
+    @requires_system_grains
+    def test_hold_unhold(self, grains):
         '''
         test holding and unholding a package
         '''
-        os_family = self.run_function('grains.item', ['os_family'])['os_family']
-        available = self.run_function('sys.doc', ['pkg.hold'])
-        version_lock = None
-        lock_pkg = 'yum-plugin-versionlock'
+        ret = None
+        if grains['os_family'] == 'RedHat':
+            # get correct plugin for dnf packages following the logic in `salt.modules.yumpkg._yum`
+            lock_pkg = 'yum-versionlock' if grains['osmajorrelease'] == '5' else 'yum-plugin-versionlock'
+            if 'fedora' in grains['os'].lower() and int(grains['osrelease']) >= 22:
+                if int(grains['osmajorrelease']) >= 26:
+                    lock_pkg = 'python{py}-dnf-plugin-versionlock'.format(py=3 if six.PY3 else 2)
+                else:
+                    lock_pkg = 'python{py}-dnf-plugins-extras-versionlock'.format(py=3 if six.PY3 else '')
+            ret = self.run_state('pkg.installed', name=lock_pkg)
 
-        if available:
-            self.run_function('pkg.install', [self.pkg])
-            if os_family == 'RedHat':
-                version_lock = self.run_function('pkg.version', [lock_pkg])
-                if not version_lock:
-                    self.run_function('pkg.install', [lock_pkg])
+        self.run_function('pkg.install', [self.pkg])
 
-            hold_ret = self.run_function('pkg.hold', [self.pkg])
-            self.assertIn(self.pkg, hold_ret)
-            self.assertTrue(hold_ret[self.pkg]['result'])
-
-            unhold_ret = self.run_function('pkg.unhold', [self.pkg])
-            self.assertIn(self.pkg, unhold_ret)
-            self.assertTrue(unhold_ret[self.pkg]['result'])
-
-            if os_family == 'RedHat':
-                if not version_lock:
-                    self.run_function('pkg.remove', [lock_pkg])
+        hold_ret = self.run_function('pkg.hold', [self.pkg])
+        if 'versionlock is not installed' in hold_ret:
             self.run_function('pkg.remove', [self.pkg])
+            self.skipTest('Versionlock could not be installed on this system: {}'.format(ret))
+        self.assertIn(self.pkg, hold_ret)
+        self.assertTrue(hold_ret[self.pkg]['result'])
 
-        else:
-            os_grain = self.run_function('grains.item', ['os'])['os']
-            self.skipTest('{0} is unavailable on {1}'.format('pkg.hold', os_grain))
+        unhold_ret = self.run_function('pkg.unhold', [self.pkg])
+        self.assertIn(self.pkg, unhold_ret)
+        self.assertTrue(unhold_ret[self.pkg]['result'])
+        self.run_function('pkg.remove', [self.pkg])
 
-    @requires_network()
     @destructiveTest
-    def test_refresh_db(self):
+    @requires_salt_modules('pkg.refresh_db')
+    @requires_network()
+    @requires_system_grains
+    def test_refresh_db(self, grains):
         '''
         test refreshing the package database
         '''
         func = 'pkg.refresh_db'
-        os_family = self.run_function('grains.item', ['os_family'])['os_family']
 
         rtag = salt.utils.pkg.rtag(self.minion_opts)
         salt.utils.pkg.write_rtag(self.minion_opts)
         self.assertTrue(os.path.isfile(rtag))
 
-        if os_family == 'RedHat':
-            ret = self.run_function(func)
+        ret = self.run_function(func)
+        if not isinstance(ret, dict):
+            self.skipTest('Upstream repo did not return coherent results: {}'.format(ret))
+
+        if grains['os_family'] == 'RedHat':
             self.assertIn(ret, (True, None))
-        elif os_family == 'Suse':
-            ret = self.run_function(func)
+        elif grains['os_family'] == 'Suse':
             if not isinstance(ret, dict):
                 self.skipTest('Upstream repo did not return coherent results. Skipping test.')
             self.assertNotEqual(ret, {})
-        elif os_family == 'Debian':
-            ret = self.run_function(func)
-            if not isinstance(ret, dict):
-                self.skipTest('{0} encountered an error: {1}'.format(func, ret))
+        elif grains['os_family'] == 'Suse':
             self.assertNotEqual(ret, {})
-            if not isinstance(ret, dict):
-                self.skipTest('Upstream repo did not return coherent results. Skipping test.')
             for source, state in ret.items():
                 self.assertIn(state, (True, False, None))
-        else:
-            os_grain = self.run_function('grains.item', ['os'])['os']
-            self.skipTest('{0} is unavailable on {1}'.format(func, os_grain))
 
         self.assertFalse(os.path.isfile(rtag))
 
     @requires_salt_modules('pkg.info_installed')
-    def test_pkg_info(self):
+    @requires_system_grains
+    def test_pkg_info(self, grains):
         '''
         Test returning useful information on Ubuntu systems.
         '''
         func = 'pkg.info_installed'
-        os_family = self.run_function('grains.item', ['os_family'])['os_family']
 
-        if os_family == 'Debian':
+        if grains['os_family'] == 'Debian':
             ret = self.run_function(func, ['bash-completion', 'dpkg'])
             keys = ret.keys()
             self.assertIn('bash-completion', keys)
             self.assertIn('dpkg', keys)
-        elif os_family == 'RedHat':
+        elif grains['os_family'] == 'RedHat':
             ret = self.run_function(func, ['rpm', 'bash'])
             keys = ret.keys()
             self.assertIn('rpm', keys)
             self.assertIn('bash', keys)
-        elif os_family == 'Suse':
+        elif grains['os_family'] == 'Suse':
             ret = self.run_function(func, ['less', 'zypper'])
             keys = ret.keys()
             self.assertIn('less', keys)
             self.assertIn('zypper', keys)
 
-    @requires_network()
     @destructiveTest
-    @skipIf(salt.utils.platform.is_windows(), 'pkg.upgrade not available on Windows')
-    def test_pkg_upgrade_has_pending_upgrades(self):
+    @requires_network()
+    @requires_salt_modules('pkg.refresh_db', 'pkg.upgrade', 'pkg.install', 'pkg.list_repo_pkgs', 'pkg.list_upgrades')
+    @requires_system_grains
+    def test_pkg_upgrade_has_pending_upgrades(self, grains):
         '''
         Test running a system upgrade when there are packages that need upgrading
         '''
         func = 'pkg.upgrade'
-        os_family = self.run_function('grains.item', ['os_family'])['os_family']
 
         # First make sure that an up-to-date copy of the package db is available
         self.run_function('pkg.refresh_db')
 
-        if os_family == 'Suse':
+        if grains['os_family'] == 'Suse':
             # This test assumes that there are multiple possible versions of a
             # package available. That makes it brittle if you pick just one
             # target, as changes in the available packages will break the test.
@@ -339,32 +320,20 @@ class PkgModuleTest(ModuleCase, SaltReturnAssertsMixin):
                 self.skipTest('No updates available for this machine.  Skipping pkg.upgrade test.')
             else:
                 args = []
-                if os_family == 'Debian':
+                if grains['os_family'] == 'Debian':
                     args = ['dist_upgrade=True']
                 ret = self.run_function(func, args)
                 self.assertNotEqual(ret, {})
 
     @destructiveTest
-    @skipIf(salt.utils.platform.is_windows(), 'minion is windows')
-    @skipIf(salt.utils.platform.is_darwin(), 'minion is mac')
-    def test_pkg_latest_version(self):
+    @requires_salt_modules('pkg.remove', 'pkg.latest_version')
+    @requires_system_grains
+    def test_pkg_latest_version(self, grains):
         '''
         Check that pkg.latest_version returns the latest version of the uninstalled package.
         The package is not installed. Only the package version is checked.
         '''
-        grains = self.run_function('grains.items')
-        remove = False
-        if salt.utils.platform.is_windows():
-            cmd_info = self.run_function('pkg.version', [self.pkg])
-            remove = False if cmd_info == '' else True
-        else:
-            cmd_info = self.run_function('pkg.info_installed', [self.pkg])
-            if cmd_info != 'ERROR: package {0} is not installed'.format(self.pkg):
-                remove = True
-
-        # remove package if its installed
-        if remove:
-            self.run_function('pkg.remove', [self.pkg])
+        self.run_state('pkg.removed', name=self.pkg)
 
         cmd_pkg = []
         if grains['os_family'] == 'RedHat':
@@ -377,5 +346,10 @@ class PkgModuleTest(ModuleCase, SaltReturnAssertsMixin):
             cmd_pkg = self.run_function('cmd.run', ['pacman -Si {0}'.format(self.pkg)])
         elif grains['os_family'] == 'Suse':
             cmd_pkg = self.run_function('cmd.run', ['zypper info {0}'.format(self.pkg)])
+        elif grains['os_family'] == 'MacOS':
+            self.skipTest('TODO the following command needs to be run as a non-root user')
+            #cmd_pkg = self.run_function('cmd.run', ['brew info {0}'.format(self.pkg)])
+        else:
+            self.skipTest('TODO: test not configured for {}'.format(grains['os_family']))
         pkg_latest = self.run_function('pkg.latest_version', [self.pkg])
         self.assertIn(pkg_latest, cmd_pkg)

tests/integration/states/test_pkgrepo.py

@@ -13,6 +13,7 @@ from tests.support.mixins import SaltReturnAssertsMixin
 from tests.support.unit import skipIf
 from tests.support.helpers import (
     destructiveTest,
+    requires_salt_modules,
     requires_system_grains,
 )
 
@@ -29,14 +30,13 @@ class PkgrepoTest(ModuleCase, SaltReturnAssertsMixin):
     '''
     pkgrepo state tests
     '''
+    @requires_salt_modules('pkgrepo.managed')
     @requires_system_grains
     def test_pkgrepo_01_managed(self, grains):
         '''
         Test adding a repo
         '''
-        os_grain = self.run_function('grains.item', ['os'])['os']
-        os_release_info = tuple(self.run_function('grains.item', ['osrelease_info'])['osrelease_info'])
-        if os_grain == 'Ubuntu' and os_release_info >= (15, 10):
+        if grains['os'] == 'Ubuntu' and grains['osrelease_info'] >= (15, 10):
             self.skipTest(
                 'The PPA used for this test does not exist for Ubuntu Wily'
                 ' (15.10) and later.'
@@ -57,13 +57,13 @@ class PkgrepoTest(ModuleCase, SaltReturnAssertsMixin):
         for state_id, state_result in six.iteritems(ret):
             self.assertSaltTrueReturn(dict([(state_id, state_result)]))
 
-    def test_pkgrepo_02_absent(self):
+    @requires_salt_modules('pkgrepo.absent')
+    @requires_system_grains
+    def test_pkgrepo_02_absent(self, grains):
         '''
         Test removing the repo from the above test
         '''
-        os_grain = self.run_function('grains.item', ['os'])['os']
-        os_release_info = tuple(self.run_function('grains.item', ['osrelease_info'])['osrelease_info'])
-        if os_grain == 'Ubuntu' and os_release_info >= (15, 10):
+        if grains['os'] == 'Ubuntu' and grains['osrelease_info'] >= (15, 10):
             self.skipTest(
                 'The PPA used for this test does not exist for Ubuntu Wily'
                 ' (15.10) and later.'
@@ -77,24 +77,21 @@ class PkgrepoTest(ModuleCase, SaltReturnAssertsMixin):
         for state_id, state_result in six.iteritems(ret):
             self.assertSaltTrueReturn(dict([(state_id, state_result)]))
 
+    @requires_salt_modules('pkgrepo.absent', 'pkgrepo.managed')
     @requires_system_grains
     def test_pkgrepo_03_with_comments(self, grains):
         '''
         Test adding a repo with comments
         '''
-        os_family = grains['os_family'].lower()
-
-        if os_family in ('redhat',):
+        if grains['os_family'] in ('redhat',):
             kwargs = {
                 'name': 'examplerepo',
                 'baseurl': 'http://example.com/repo',
                 'enabled': False,
                 'comments': ['This is a comment']
             }
-        elif os_family in ('debian',):
+        elif grains['os_family'] in ('debian',):
             self.skipTest('Debian/Ubuntu test case needed')
-        else:
-            self.skipTest("No test case for os_family '{0}'".format(os_family))
 
         try:
             # Run the state to add the repo
@@ -130,6 +127,7 @@ class PkgrepoTest(ModuleCase, SaltReturnAssertsMixin):
             # Clean up
             self.run_state('pkgrepo.absent', name=kwargs['name'])
 
+    @requires_salt_modules('pkgrepo.managed')
     @requires_system_grains
     def test_pkgrepo_04_apt_with_architectures(self, grains):
         '''

tests/multimaster/__init__.py

@@ -191,7 +191,7 @@ class MultimasterTestDaemon(TestDaemon):
                 daemon_class=SaltMaster,
                 bin_dir_path=SCRIPT_DIR,
                 fail_hard=True,
-                start_timeout=60)
+                start_timeout=120)
             sys.stdout.write(
                 '\r{0}\r'.format(
                     ' ' * getattr(self.parser.options, 'output_columns', PNUM)
@@ -235,7 +235,7 @@ class MultimasterTestDaemon(TestDaemon):
                 daemon_class=SaltMaster,
                 bin_dir_path=SCRIPT_DIR,
                 fail_hard=True,
-                start_timeout=60)
+                start_timeout=120)
             sys.stdout.write(
                 '\r{0}\r'.format(
                     ' ' * getattr(self.parser.options, 'output_columns', PNUM)
@@ -272,7 +272,7 @@ class MultimasterTestDaemon(TestDaemon):
                 daemon_class=SaltMinion,
                 bin_dir_path=SCRIPT_DIR,
                 fail_hard=True,
-                start_timeout=60)
+                start_timeout=120)
             sys.stdout.write(
                 '\r{0}\r'.format(
                     ' ' * getattr(self.parser.options, 'output_columns', PNUM)
@@ -309,7 +309,7 @@ class MultimasterTestDaemon(TestDaemon):
                 daemon_class=SaltMinion,
                 bin_dir_path=SCRIPT_DIR,
                 fail_hard=True,
-                start_timeout=60)
+                start_timeout=120)
             sys.stdout.write(
                 '\r{0}\r'.format(
                     ' ' * getattr(self.parser.options, 'output_columns', PNUM)