Merge branch '2017.7' into fix_win_service

.ci/kitchen-windows2016-py2

@@ -0,0 +1,73 @@
+pipeline {
+    agent { label 'kitchen-slave' }
+    options {
+        timestamps()
+        ansiColor('xterm')
+    }
+    environment {
+        SALT_KITCHEN_PLATFORMS = "/var/jenkins/workspace/platforms.yml"
+        SALT_KITCHEN_DRIVER = "/var/jenkins/workspace/driver.yml"
+        PATH = "/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin"
+        RBENV_VERSION = "2.4.2"
+        TEST_SUITE = "py2"
+        TEST_PLATFORM = "windows-2016"
+        PY_COLORS = 1
+    }
+    stages {
+        stage('github-pending') {
+            steps {
+                githubNotify credentialsId: 'test-jenkins-credentials',
+                    description: "running ${TEST_SUITE}-${TEST_PLATFORM}...",
+                    status: 'PENDING',
+                    context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}"
+            }
+        }
+        stage('setup') {
+            steps {
+                sh 'bundle install --with ec2 windows --without opennebula docker'
+            }
+        }
+        stage('run kitchen') {
+            steps {
+                script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
+                    sshagent(credentials: ['jenkins-testing-ssh-key']) {
+                        sh 'ssh-add ~/.ssh/jenkins-testing.pem'
+                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM'
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM'
+                    }
+                }}
+            }
+            post {
+                always {
+                    script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
+                        sshagent(credentials: ['jenkins-testing-ssh-key']) {
+                            sh 'ssh-add ~/.ssh/jenkins-testing.pem'
+                            sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM'
+                        }
+                    }}
+                    archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml'
+                    archiveArtifacts artifacts: 'artifacts/logs/minion'
+                    archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log'
+                }
+            }
+        }
+    }
+    post {
+        always {
+            junit 'artifacts/xml-unittests-output/*.xml'
+            cleanWs()
+        }
+        success {
+            githubNotify credentialsId: 'test-jenkins-credentials',
+                description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed",
+                status: 'SUCCESS',
+                context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}"
+        }
+        failure {
+            githubNotify credentialsId: 'test-jenkins-credentials',
+                description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed",
+                status: 'FAILURE',
+                context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}"
+        }
+    }
+}

.ci/kitchen-windows2016-py3

@@ -0,0 +1,73 @@
+pipeline {
+    agent { label 'kitchen-slave' }
+    options {
+        timestamps()
+        ansiColor('xterm')
+    }
+    environment {
+        SALT_KITCHEN_PLATFORMS = "/var/jenkins/workspace/platforms.yml"
+        SALT_KITCHEN_DRIVER = "/var/jenkins/workspace/driver.yml"
+        PATH = "/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin"
+        RBENV_VERSION = "2.4.2"
+        TEST_SUITE = "py3"
+        TEST_PLATFORM = "windows-2016"
+        PY_COLORS = 1
+    }
+    stages {
+        stage('github-pending') {
+            steps {
+                githubNotify credentialsId: 'test-jenkins-credentials',
+                    description: "running ${TEST_SUITE}-${TEST_PLATFORM}...",
+                    status: 'PENDING',
+                    context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}"
+            }
+        }
+        stage('setup') {
+            steps {
+                sh 'bundle install --with ec2 windows --without opennebula docker'
+            }
+        }
+        stage('run kitchen') {
+            steps {
+                script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
+                    sshagent(credentials: ['jenkins-testing-ssh-key']) {
+                        sh 'ssh-add ~/.ssh/jenkins-testing.pem'
+                        sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM'
+                        sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM'
+                    }
+                }}
+            }
+            post {
+                always {
+                    script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
+                        sshagent(credentials: ['jenkins-testing-ssh-key']) {
+                            sh 'ssh-add ~/.ssh/jenkins-testing.pem'
+                            sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM'
+                        }
+                    }}
+                    archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml'
+                    archiveArtifacts artifacts: 'artifacts/logs/minion'
+                    archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log'
+                }
+            }
+        }
+    }
+    post {
+        always {
+            junit 'artifacts/xml-unittests-output/*.xml'
+            cleanWs()
+        }
+        success {
+            githubNotify credentialsId: 'test-jenkins-credentials',
+                description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed",
+                status: 'SUCCESS',
+                context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}"
+        }
+        failure {
+            githubNotify credentialsId: 'test-jenkins-credentials',
+                description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed",
+                status: 'FAILURE',
+                context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}"
+        }
+    }
+}

.ci/lint

@@ -20,8 +20,15 @@ pipeline {
         }
         stage('setup') {
             steps {
-                sh 'eval "$(pyenv init -)"; pyenv install 2.7.14 || echo "We already have this python."; pyenv local 2.7.14; pyenv shell 2.7.14'
-                sh 'eval "$(pyenv init -)"; pip install tox'
+                sh '''
+                eval "$(pyenv init -)"
+                pyenv --version
+                pyenv install --skip-existing 2.7.14
+                pyenv local 2.7.14
+                pyenv shell 2.7.14
+                python --version
+                pip install tox
+                '''
             }
         }
         stage('linting') {
@@ -29,13 +36,32 @@ pipeline {
             parallel {
                 stage('salt linting') {
                     steps {
-                        sh 'eval "$(pyenv init - --no-rehash)"; tox -e pylint-salt $(find salt/ -name "*.py" -exec git diff --name-only "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME" setup.py {} +) | tee pylint-report.xml'
+                        sh '''
+                        eval "$(pyenv init - --no-rehash)"
+                        _FILES="$(find salt/ -name "*.py" -exec git diff --name-only "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME" {} +)"
+                        _FILES="$_FILES $(git diff --name-only "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME" setup.py)"
+                        if [[ -z ${_FILES} ]]; then
+                            echo "No pylint run, no changes found in the files"
+                            echo "empty" pylint-reports.xml
+                        else
+                            tox -e pylint-salt ${_FILES} | tee pylint-report.xml
+                        fi
+                        '''
                         archiveArtifacts artifacts: 'pylint-report.xml'
                     }
                 }
                 stage('test linting') {
                     steps {
-                        sh 'eval "$(pyenv init - --no-rehash)"; tox -e pylint-tests $(find tests/ -name "*.py" -exec git diff --name-only "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME" {} +) | tee pylint-report-tests.xml'
+                        sh '''
+                        eval "$(pyenv init - --no-rehash)"
+                        _FILES="$(find tests/ -name "*.py" -exec git diff --name-only "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME" setup.py {} +)"
+                        if [[ -z ${_FILES} ]]; then
+                            echo "No pylint run, no changes found in the files"
+                            touch pylint-report-tests.xml
+                        else
+                            tox -e pylint-tests ${_FILES} | tee pylint-report-tests.xml
+                        fi
+                        '''
                         archiveArtifacts artifacts: 'pylint-report-tests.xml'
                     }
                 }

conf/master

@@ -125,7 +125,7 @@
 # The master can take a while to start up when lspci and/or dmidecode is used
 # to populate the grains for the master. Enable if you want to see GPU hardware
 # data for your master.
-# enable_gpu_grains: True
+# enable_gpu_grains: False
 
 # The master maintains a job cache. While this is a great addition, it can be
 # a burden on the master for larger deployments (over 5000 minions).

conf/minion

@@ -148,6 +148,11 @@
 # Set the directory used to hold unix sockets.
 #sock_dir: /var/run/salt/minion
 
+# The minion can take a while to start up when lspci and/or dmidecode is used
+# to populate the grains for the minion. Set this to False if you do not need
+# GPU hardware grains for your minion.
+# enable_gpu_grains: True
+
 # Set the default outputter used by the salt-call command. The default is
 # "nested".
 #output: nested

conf/suse/master

@@ -127,7 +127,7 @@ syndic_user: salt
 # The master can take a while to start up when lspci and/or dmidecode is used
 # to populate the grains for the master. Enable if you want to see GPU hardware
 # data for your master.
-# enable_gpu_grains: True
+# enable_gpu_grains: False
 
 # The master maintains a job cache. While this is a great addition, it can be
 # a burden on the master for larger deployments (over 5000 minions).

doc/man/salt.7

@@ -5261,7 +5261,7 @@ sock_dir: /var/run/salt/master
 .UNINDENT
 .SS \fBenable_gpu_grains\fP
 .sp
-Default: \fBFalse\fP
+Default: \fBTrue\fP
 .sp
 Enable GPU hardware data for your master. Be aware that the master can
 take a while to start up when lspci and/or dmidecode is used to populate the
@@ -14347,7 +14347,7 @@ and \fBmine_functions\fP\&.
 # The master can take a while to start up when lspci and/or dmidecode is used
 # to populate the grains for the master. Enable if you want to see GPU hardware
 # data for your master.
-# enable_gpu_grains: True
+# enable_gpu_grains: False
 
 # The master maintains a job cache. While this is a great addition, it can be
 # a burden on the master for larger deployments (over 5000 minions).

doc/ref/configuration/master.rst

@@ -446,6 +446,10 @@ Enable GPU hardware data for your master. Be aware that the master can
 take a while to start up when lspci and/or dmidecode is used to populate the
 grains for the master.
 
+.. code-block:: yaml
+
+    enable_gpu_grains: True
+
 .. conf_master:: job_cache
 
 ``job_cache``

doc/ref/configuration/minion.rst

@@ -707,6 +707,22 @@ The directory where Unix sockets will be kept.
 
     sock_dir: /var/run/salt/minion
 
+.. conf_minion:: enable_gpu_grains
+
+``enable_gpu_grains``
+---------------------
+
+Default: ``True``
+
+Enable GPU hardware data for your master. Be aware that the minion can
+take a while to start up when lspci and/or dmidecode is used to populate the
+grains for the minion, so this can be set to ``False`` if you do not need these
+grains.
+
+.. code-block:: yaml
+
+    enable_gpu_grains: False
+
 .. conf_minion:: outputter_dirs
 
 ``outputter_dirs``

pkg/osx/build_env.sh

@@ -161,7 +161,7 @@ sudo -H $MAKE install
 ############################################################################
 echo -n -e "\033]0;Build_Env: libsodium\007"
 
-PKGURL="https://download.libsodium.org/libsodium/releases/libsodium-1.0.13.tar.gz"
+PKGURL="https://download.libsodium.org/libsodium/releases/old/libsodium-1.0.13.tar.gz"
 PKGDIR="libsodium-1.0.13"
 
 download $PKGURL

salt/config/__init__.py

@@ -414,6 +414,9 @@ VALID_OPTS = {
     # Tell the loader to attempt to import *.pyx cython files if cython is available
     'cython_enable': bool,
 
+    # Whether or not to load grains for the GPU
+    'enable_gpu_grains': bool,
+
     # Tell the loader to attempt to import *.zip archives
     'enable_zip_modules': bool,
 
@@ -1264,6 +1267,7 @@ DEFAULT_MINION_OPTS = {
     'test': False,
     'ext_job_cache': '',
     'cython_enable': False,
+    'enable_gpu_grains': True,
     'enable_zip_modules': False,
     'state_verbose': True,
     'state_output': 'full',

salt/fileserver/__init__.py

@@ -490,6 +490,15 @@ class Fileserver(object):
             return ret
         return list(ret)
 
+    def file_envs(self, load=None):
+        '''
+        Return environments for all backends for requests from fileclient
+        '''
+        if load is None:
+            load = {}
+        load.pop('cmd', None)
+        return self.envs(**load)
+
     def init(self, back=None):
         '''
         Initialize the backend, only do so if the fs supports an init function

salt/loader.py

@@ -1357,11 +1357,11 @@ class LazyLoader(salt.utils.lazy.LazyDict):
         '''
         if '__grains__' not in self.pack:
             self.context_dict['grains'] = opts.get('grains', {})
-            self.pack['__grains__'] = salt.utils.context.NamespacedDictWrapper(self.context_dict, 'grains', override_name='grains')
+            self.pack['__grains__'] = salt.utils.context.NamespacedDictWrapper(self.context_dict, 'grains')
 
         if '__pillar__' not in self.pack:
             self.context_dict['pillar'] = opts.get('pillar', {})
-            self.pack['__pillar__'] = salt.utils.context.NamespacedDictWrapper(self.context_dict, 'pillar', override_name='pillar')
+            self.pack['__pillar__'] = salt.utils.context.NamespacedDictWrapper(self.context_dict, 'pillar')
 
         mod_opts = {}
         for key, val in list(opts.items()):

salt/master.py

@@ -963,7 +963,7 @@ class AESFuncs(object):
         self._file_list_emptydirs = self.fs_.file_list_emptydirs
         self._dir_list = self.fs_.dir_list
         self._symlink_list = self.fs_.symlink_list
-        self._file_envs = self.fs_.envs
+        self._file_envs = self.fs_.file_envs
 
     def __verify_minion(self, id_, token):
         '''

salt/modules/consul.py

@@ -2164,7 +2164,7 @@ def acl_info(consul_url=None, **kwargs):
     function = 'acl/info/{0}'.format(kwargs['id'])
     ret = _query(consul_url=consul_url,
                  data=data,
-                 method='PUT',
+                 method='GET',
                  function=function)
     return ret
 

salt/modules/cp.py

@@ -296,6 +296,19 @@ def get_file(path,
                 gzip)
 
 
+def envs():
+    '''
+    List available environments for fileserver
+
+    CLI Example
+
+    .. code-block:: bash
+
+        salt '*' cp.envs
+    '''
+    return _client().envs()
+
+
 def get_template(path,
                  dest,
                  template='jinja',

salt/modules/mac_user.py

@@ -467,7 +467,8 @@ def list_users():
 
         salt '*' user.list_users
     '''
-    return sorted([user.pw_name for user in pwd.getpwall()])
+    users = _dscl(['/users'], 'list')['stdout']
+    return users.split()
 
 
 def rename(name, new_name):

salt/modules/win_lgpo.py

@@ -4896,7 +4896,7 @@ def _lookup_admin_template(policy_name,
 def get_policy_info(policy_name,
                     policy_class,
                     adml_language='en-US'):
-    '''
+    r'''
     Returns information about a specified policy
 
     Args:
@@ -4915,6 +4915,124 @@ def get_policy_info(policy_name,
     .. code-block:: bash
 
         salt '*' lgpo.get_policy_info 'Maximum password age' machine
+
+    You can use ``lgpo.get_policy_info`` to get all the possible names that
+    could be used in a state file or from the command line (along with elements
+    that need to be set/etc). The key is to match the text you see in the
+    ``gpedit.msc`` gui exactly, including quotes around words or phrases. The
+    "full path" style is really only needed when there are multiple policies
+    that use the same base name. For example, ``Access data sources across
+    domains`` exists in ~10 different paths. If you put that through
+    ``get_policy_info`` you'll get back a message that it is used for multiple
+    policies and you need to be more specific.
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt-call --local lgpo.get_policy_info ShellRemoveOrderPrints_2 machine
+
+        local:
+            ----------
+            message:
+            policy_aliases:
+                - Turn off the "Order Prints" picture task
+                - ShellRemoveOrderPrints_2
+                - System\Internet Communication Management\Internet Communication settings\Turn off the "Order Prints" picture task
+            policy_class:
+                machine
+            policy_elements:
+            policy_found:
+                True
+            policy_name:
+                ShellRemoveOrderPrints_2
+            rights_assignment:
+                False
+
+    Escaping can get tricky in cmd/Powershell. The following is an example of
+    escaping in Powershell using backquotes:
+
+    .. code-block:: bash
+
+        PS>salt-call --local lgpo.get_policy_info "Turn off the `\`"Order Prints`\`" picture task" machine
+
+        local:
+            ----------
+            message:
+            policy_aliases:
+                - Turn off the "Order Prints" picture task
+                - ShellRemoveOrderPrints_2
+                - System\Internet Communication Management\Internet Communication settings\Turn off the "Order Prints" picture task
+            policy_class:
+                machine
+            policy_elements:
+            policy_found:
+                True
+            policy_name:
+                Turn off the "Order Prints" picture task
+            rights_assignment:
+                False
+
+    This function can then be used to get the options available for specifying
+    Group Policy Objects to be used in state files. Based on the above any of
+    these *should* be usable:
+
+    .. code-block:: bash
+
+        internet_communications_settings:
+          lgpo.set:
+            - computer_policy:
+                Turn off the "Order Prints" picture task: Enabled
+
+    .. code-block:: bash
+
+        internet_communications_settings:
+          lgpo.set:
+            - computer_policy:
+                ShellRemoveOrderPrints_2: Enabled
+
+    When using the full path, it might be a good idea to use single quotes
+    around the path:
+
+    .. code-block:: bash
+
+        internet_communications_settings:
+          lgpo.set:
+            - computer_policy:
+                'System\Internet Communication Management\Internet Communication settings\Turn off the "Order Prints" picture task': 'Enabled'
+
+    If you struggle to find the policy from ``get_policy_info`` using the name
+    as you see in ``gpedit.msc``, the names such as "ShellRemoveOrderPrints_2"
+    come from the ``.admx`` files. If you know nothing about ``.admx/.adml``
+    relationships (ADML holds what you see in the GUI, ADMX holds the more
+    technical details), then this may be a little bit too much info, but here is
+    an example with the above policy using Powershell:
+
+
+    .. code-block:: bash
+
+        PS>Get-ChildItem -Path C:\Windows\PolicyDefinitions -Recurse -Filter *.adml | Select-String "Order Prints"
+
+        C:\windows\PolicyDefinitions\en-US\ICM.adml:152:      <string id="ShellRemoveOrderPrints">Turn off the "Order Prints" picture task</string>
+        C:\windows\PolicyDefinitions\en-US\ICM.adml:153:      <string id="ShellRemoveOrderPrints_Help">This policy setting specifies whether the "Order Prints Online" task is available from Picture Tasks in Windows folders.
+        C:\windows\PolicyDefinitions\en-US\ICM.adml:155:The Order Prints Online Wizard is used to download a list of providers and allow users to order prints online.
+        C:\windows\PolicyDefinitions\en-US\ICM.adml:157:If you enable this policy setting, the task "Order Prints Online" is removed from Picture Tasks in File Explorer folders.
+
+    From this grep, we can see id "ShellRemoveOrderPrints" is the ID of the
+    string used to describe this policy, then we search for it in the ADMX:
+
+    .. code-block:: bash
+
+        PS>Get-ChildItem -Path C:\Windows\PolicyDefinitions -Recurse -Filter *.admx | Select-String "ShellRemoveOrderPrints"
+
+        C:\windows\PolicyDefinitions\ICM.admx:661:    <policy name="ShellRemoveOrderPrints_1" class="User" displayName="$(string.ShellRemoveOrderPrints)" explainText="$(string.ShellRemoveOrderPrints_Help)" key="Software\Microsoft\Windows\CurrentVersion\Policies\Explorer" valueName="NoOnlinePrintsWizard">
+        C:\windows\PolicyDefinitions\ICM.admx:671:    <policy name="ShellRemoveOrderPrints_2" class="Machine" displayName="$(string.ShellRemoveOrderPrints)" explainText="$(string.ShellRemoveOrderPrints_Help)" key="Software\Microsoft\Windows\CurrentVersion\Policies\Explorer" valueName="NoOnlinePrintsWizard">
+
+    Now we have two to pick from. And if you notice the ``class="Machine"`` and
+    ``class="User"`` (which details if it is a computer policy or user policy
+    respectively) the ``ShellRemoveOrderPrints_2`` is the "short name" we could
+    use to pass through ``get_policy_info`` to see what the module itself is
+    expecting.
     '''
     # return the possible policy names and element names
     ret = {'policy_name': policy_name,

salt/modules/win_repo.py

@@ -26,7 +26,8 @@ from salt.runners.winrepo import (
     genrepo as _genrepo,
     update_git_repos as _update_git_repos,
     PER_REMOTE_OVERRIDES,
-    PER_REMOTE_ONLY
+    PER_REMOTE_ONLY,
+    GLOBAL_ONLY
 )
 from salt.ext import six
 try:

salt/modules/win_system.py

@@ -303,7 +303,7 @@ def shutdown(message=None, timeout=5, force_close=True, reboot=False,  # pylint:
                                         force_close, reboot)
         return True
     except pywintypes.error as exc:
-        (number, context, message) = exc
+        (number, context, message) = exc.args
         log.error('Failed to shutdown the system')
         log.error('nbr: {0}'.format(number))
         log.error('ctx: {0}'.format(context))
@@ -346,7 +346,7 @@ def shutdown_abort():
         win32api.AbortSystemShutdown('127.0.0.1')
         return True
     except pywintypes.error as exc:
-        (number, context, message) = exc
+        (number, context, message) = exc.args
         log.error('Failed to abort system shutdown')
         log.error('nbr: {0}'.format(number))
         log.error('ctx: {0}'.format(context))
@@ -485,7 +485,7 @@ def set_computer_desc(desc=None):
     try:
         win32net.NetServerSetInfo(None, 101, system_info)
     except win32net.error as exc:
-        (number, context, message) = exc
+        (number, context, message) = exc.args
         log.error('Failed to update system')
         log.error('nbr: {0}'.format(number))
         log.error('ctx: {0}'.format(context))
@@ -1016,7 +1016,7 @@ def set_system_date_time(years=None,
     try:
         date_time = win32api.GetLocalTime()
     except win32api.error as exc:
-        (number, context, message) = exc
+        (number, context, message) = exc.args
         log.error('Failed to get local time')
         log.error('nbr: {0}'.format(number))
         log.error('ctx: {0}'.format(context))

salt/modules/win_useradd.py

@@ -164,7 +164,6 @@ def add(name,
     try:
         win32net.NetUserAdd(None, 1, user_info)
     except win32net.error as exc:
-        (number, context, message) = exc
         log.error('Failed to create user {0}'.format(name))
         log.error('nbr: {0}'.format(exc.winerror))
         log.error('ctx: {0}'.format(exc.funcname))
@@ -266,7 +265,6 @@ def update(name,
     try:
         user_info = win32net.NetUserGetInfo(None, name, 4)
     except win32net.error as exc:
-        (number, context, message) = exc
         log.error('Failed to update user {0}'.format(name))
         log.error('nbr: {0}'.format(exc.winerror))
         log.error('ctx: {0}'.format(exc.funcname))
@@ -326,7 +324,6 @@ def update(name,
     try:
         win32net.NetUserSetInfo(None, name, 4, user_info)
     except win32net.error as exc:
-        (number, context, message) = exc
         log.error('Failed to update user {0}'.format(name))
         log.error('nbr: {0}'.format(exc.winerror))
         log.error('ctx: {0}'.format(exc.funcname))
@@ -415,7 +412,7 @@ def delete(name,
             sid = getUserSid(name)
             win32profile.DeleteProfile(sid)
         except pywintypes.error as exc:
-            (number, context, message) = exc
+            (number, context, message) = exc.args
             if number == 2:  # Profile Folder Not Found
                 pass
             else:
@@ -429,7 +426,6 @@ def delete(name,
     try:
         win32net.NetUserDel(None, name)
     except win32net.error as exc:
-        (number, context, message) = exc
         log.error('Failed to delete user {0}'.format(name))
         log.error('nbr: {0}'.format(exc.winerror))
         log.error('ctx: {0}'.format(exc.funcname))

salt/utils/yamldumper.py

@@ -17,6 +17,7 @@ except ImportError:
 
 import yaml
 import collections
+import salt.utils.context
 from salt.utils.odict import OrderedDict
 
 try:
@@ -54,6 +55,14 @@ SafeOrderedDumper.add_representer(
     collections.defaultdict,
     yaml.representer.SafeRepresenter.represent_dict
 )
+OrderedDumper.add_representer(
+    salt.utils.context.NamespacedDictWrapper,
+    yaml.representer.SafeRepresenter.represent_dict
+)
+SafeOrderedDumper.add_representer(
+    salt.utils.context.NamespacedDictWrapper,
+    yaml.representer.SafeRepresenter.represent_dict
+)
 
 if HAS_IOFLO:
     OrderedDumper.add_representer(odict, represent_ordereddict)

tests/integration/modules/test_archive.py

@@ -65,6 +65,10 @@ class ArchiveTest(ModuleCase):
         else:
             filename = 'file'
         with salt.utils.fopen(os.path.join(self.src, filename), 'wb') as theorem:
+            if six.PY3 and salt.utils.is_windows():
+                encoding = 'utf-8'
+            else:
+                encoding = None
             theorem.write(salt.utils.to_bytes(textwrap.dedent('''\
                 Compression theorem of computational complexity theory:
 
@@ -82,7 +86,7 @@ class ArchiveTest(ModuleCase):
                 and
 
                     $\\mathrm C(φ_i) ⊊ \\mathrm{C}(φ_{f(i)})$.
-            ''')))
+            '''), encoding=encoding))
 
         # Create destination
         os.makedirs(self.dst)
@@ -124,10 +128,10 @@ class ArchiveTest(ModuleCase):
         dir_in_ret = None
         file_in_ret = None
         for line in ret:
-            if normdir(self.src) in line \
-            and not normdir(self.src_file) in line:
+            if normdir(self.src) in os.path.normcase(line) \
+            and not normdir(self.src_file) in os.path.normcase(line):
                 dir_in_ret = True
-            if normdir(self.src_file) in line:
+            if normdir(self.src_file) in os.path.normcase(line):
                 file_in_ret = True
 
         # Assert number of lines, reporting of source directory and file
@@ -250,7 +254,7 @@ class ArchiveTest(ModuleCase):
         # Test create archive
         ret = self.run_function('archive.unzip', [self.arch, self.dst])
         self.assertTrue(isinstance(ret, list), six.text_type(ret))
-        self._assert_artifacts_in_ret(ret, unix_sep=True if six.PY2 else False)
+        self._assert_artifacts_in_ret(ret, unix_sep=False)
 
         self._tear_down()
 

tests/integration/modules/test_cp.py

@@ -614,3 +614,6 @@ class CPModuleTest(ModuleCase):
             self.assertTrue(os.path.isfile(tgt_cache_file), 'File was not cached on the master')
         finally:
             os.unlink(tgt_cache_file)
+
+    def test_envs(self):
+        self.assertEqual(self.run_function('cp.envs'), ['base', 'prod'])

tests/integration/modules/test_system.py

@@ -11,7 +11,7 @@ import subprocess
 # Import Salt Testing libs
 from tests.support.case import ModuleCase
 from tests.support.unit import skipIf
-from tests.support.helpers import destructiveTest, skip_if_not_root
+from tests.support.helpers import destructiveTest, skip_if_not_root, flaky
 
 # Import salt libs
 import salt.utils
@@ -251,6 +251,7 @@ class SystemModuleTest(ModuleCase):
         self.assertTrue(self._same_times(time_now, cmp_time), msg=msg)
         self._test_hwclock_sync()
 
+    @flaky
     @destructiveTest
     @skip_if_not_root
     def test_set_system_time(self):

tests/integration/modules/test_useradd.py

@@ -268,8 +268,8 @@ class UseraddModuleTestWindows(ModuleCase):
         self._add_group()
         self.run_function('user.addgroup', [self.user_name, self.group_name])
         self.assertIn(self.group_name, self.run_function('user.list_groups', [self.user_name]))
-        self.run_function('user.removegroup', [self.group_name])
-        self.assertIn(self.group_name, self.run_function('user.list_groups', [self.user_name]))
+        self.run_function('user.removegroup', [self.user_name, self.group_name])
+        self.assertNotIn(self.group_name, self.run_function('user.list_groups', [self.user_name]))
 
     def test_user_rename(self):
         '''

tests/integration/output/test_output.py

@@ -7,6 +7,7 @@
 from __future__ import absolute_import
 import os
 import traceback
+import yaml
 
 # Import Salt Testing Libs
 from tests.support.case import ShellCase
@@ -81,6 +82,20 @@ class OutputReturnTest(ShellCase):
         ret = self.run_call('test.ping --out=yaml')
         self.assertEqual(ret, expected)
 
+    def test_output_yaml_namespaced_dict_wrapper(self):
+        '''
+        Tests the ability to dump a NamespacedDictWrapper instance, as used in
+        magic dunders like __grains__ and __pillar__
+
+        See https://github.com/saltstack/salt/issues/49269
+        '''
+        dumped_yaml = '\n'.join(self.run_call('grains.items --out=yaml'))
+        loaded_yaml = yaml.load(dumped_yaml)
+        # We just want to check that the dumped YAML loades as a dict with a
+        # single top-level key, we don't care about the real contents.
+        assert isinstance(loaded_yaml, dict)
+        assert list(loaded_yaml) == ['local']
+
     def test_output_unicodebad(self):
         '''
         Tests outputter reliability with utf8

tests/unit/modules/test_mac_user.py

@@ -312,6 +312,11 @@ class MacUserTestCase(TestCase, LoaderModuleMockMixin):
         '''
         Tests the list of all users
         '''
-        with patch('pwd.getpwall', MagicMock(return_value=self.mock_pwall)):
-            ret = ['_amavisd', '_appleevents', '_appowner']
-            self.assertEqual(mac_user.list_users(), ret)
+        expected = ['spongebob', 'patrick', 'squidward']
+        mock_run = MagicMock(return_value={'pid': 4948,
+                                           'retcode': 0,
+                                           'stderr': '',
+                                           'stdout': '\n'.join(expected)})
+        with patch.dict(mac_user.__grains__, {'osrelease_info': (10, 9, 1)}), \
+                patch.dict(mac_user.__salt__, {'cmd.run_all': mock_run}):
+            self.assertEqual(mac_user.list_users(), expected)

tests/unit/serializers/test_serializers.py

@@ -24,7 +24,7 @@ from salt.utils.odict import OrderedDict
 # Import test support libs
 from tests.support.helpers import flaky
 
-SKIP_MESSAGE = '%s is unavailable, do prerequisites have been met?'
+SKIP_MESSAGE = '%s is unavailable, have prerequisites been met?'
 
 
 @flaky(condition=six.PY3)
@@ -93,6 +93,7 @@ class TestSerializers(TestCase):
 
     @skipIf(not yaml.available, SKIP_MESSAGE % 'yaml')
     @skipIf(not yamlex.available, SKIP_MESSAGE % 'sls')
+    @flaky
     def test_compare_sls_vs_yaml_with_jinja(self):
         tpl = '{{ data }}'
         env = jinja2.Environment()

tests/whitelist.txt

@@ -9,7 +9,6 @@ integration.doc.test_man
 integration.grains.test_core
 integration.loader.test_ext_grains
 integration.loader.test_ext_modules
-integration.logging.test_jid_logging
 integration.minion.test_blackout
 integration.minion.test_pillar
 integration.minion.test_timeout
@@ -41,7 +40,7 @@ integration.modules.test_sysmod
 integration.modules.test_system
 integration.modules.test_test
 integration.modules.test_useradd
-integration.modules.test_win_autoruns
+integration.modules.test_autoruns
 integration.modules.test_win_dns_client
 integration.modules.test_win_pkg
 integration.reactor.test_reactor