From 7cdba27c2b99e1dfd143a752a002134e4196f483 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Fri, 24 Aug 2018 16:09:40 -0700 Subject: [PATCH 01/26] Add ci scripts for windows PR builds --- .ci/kitchen-windows2016-py2 | 73 +++++++++++++++++++++++++++++++++++++ .ci/kitchen-windows2016-py3 | 73 +++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 .ci/kitchen-windows2016-py2 create mode 100644 .ci/kitchen-windows2016-py3 diff --git a/.ci/kitchen-windows2016-py2 b/.ci/kitchen-windows2016-py2 new file mode 100644 index 00000000000..d0e7320975b --- /dev/null +++ b/.ci/kitchen-windows2016-py2 @@ -0,0 +1,73 @@ +pipeline { + agent { label 'kitchen-slave' } + options { + timestamps() + ansiColor('xterm') + } + environment { + SALT_KITCHEN_PLATFORMS = "/var/jenkins/workspace/platforms.yml" + SALT_KITCHEN_DRIVER = "/var/jenkins/workspace/driver.yml" + PATH = "/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin" + RBENV_VERSION = "2.4.2" + TEST_SUITE = "py2" + TEST_PLATFORM = "windows-2016" + PY_COLORS = 1 + } + stages { + stage('github-pending') { + steps { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + stage('setup') { + steps { + sh 'bundle install --with ec2 windows --without opennebula docker' + } + } + stage('run kitchen') { + steps { + script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + }} + } + post { + always { + script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + } + }} + archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' + archiveArtifacts artifacts: 'artifacts/logs/minion' + archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + } + } + } + post { + always { + junit 'artifacts/xml-unittests-output/*.xml' + cleanWs() + } + success { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + failure { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } +} diff --git a/.ci/kitchen-windows2016-py3 b/.ci/kitchen-windows2016-py3 new file mode 100644 index 00000000000..11c36614477 --- /dev/null +++ b/.ci/kitchen-windows2016-py3 @@ -0,0 +1,73 @@ +pipeline { + agent { label 'kitchen-slave' } + options { + timestamps() + ansiColor('xterm') + } + environment { + SALT_KITCHEN_PLATFORMS = "/var/jenkins/workspace/platforms.yml" + SALT_KITCHEN_DRIVER = "/var/jenkins/workspace/driver.yml" + PATH = "/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin" + RBENV_VERSION = "2.4.2" + TEST_SUITE = "py3" + TEST_PLATFORM = "windows-2016" + PY_COLORS = 1 + } + stages { + stage('github-pending') { + steps { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + stage('setup') { + steps { + sh 'bundle install --with ec2 windows --without opennebula docker' + } + } + stage('run kitchen') { + steps { + script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + }} + } + post { + always { + script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + } + }} + archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' + archiveArtifacts artifacts: 'artifacts/logs/minion' + archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + } + } + } + post { + always { + junit 'artifacts/xml-unittests-output/*.xml' + cleanWs() + } + success { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + failure { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } +} From 74b05ef53085b0ec6604b09f07a3536edcf115c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Jouannet?= Date: Tue, 21 Aug 2018 17:49:47 +0200 Subject: [PATCH 02/26] fix HTTP method for acl_info --- salt/modules/consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/consul.py b/salt/modules/consul.py index 48a99974a67..57d3fb8a1a9 100644 --- a/salt/modules/consul.py +++ b/salt/modules/consul.py @@ -2164,7 +2164,7 @@ def acl_info(consul_url=None, **kwargs): function = 'acl/info/{0}'.format(kwargs['id']) ret = _query(consul_url=consul_url, data=data, - method='PUT', + method='GET', function=function) return ret From bc52f7c1bfc6bbd00c2ae8c583e13902d8a7a6e4 Mon Sep 17 00:00:00 2001 From: William Giokas <1007380@gmail.com> Date: Fri, 10 Aug 2018 17:27:58 -0600 Subject: [PATCH 03/26] Stop running lint on all files when no changes Stops our lint from running on all of the files in `salt/` or `test/` when no files were modified. By default, when tox is told to run with no positional arguments, it'll run on all files we'd look at. When run with positional arguments it'll run only on those files. This makes it so that tox, by default, will only run lint checks on commits that actually change the files in `salt/` or `test/`. --- .ci/lint | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/.ci/lint b/.ci/lint index a820c149de7..885174b0bfc 100644 --- a/.ci/lint +++ b/.ci/lint @@ -20,8 +20,15 @@ pipeline { } stage('setup') { steps { - sh 'eval "$(pyenv init -)"; pyenv install 2.7.14 || echo "We already have this python."; pyenv local 2.7.14; pyenv shell 2.7.14' - sh 'eval "$(pyenv init -)"; pip install tox' + sh ''' + eval "$(pyenv init -)" + pyenv --version + pyenv install --skip-existing 2.7.14 + pyenv local 2.7.14 + pyenv shell 2.7.14 + python --version + pip install tox + ''' } } stage('linting') { @@ -29,13 +36,32 @@ pipeline { parallel { stage('salt linting') { steps { - sh 'eval "$(pyenv init - --no-rehash)"; tox -e pylint-salt $(find salt/ -name "*.py" -exec git diff --name-only "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME" setup.py {} +) | tee pylint-report.xml' + sh ''' + eval "$(pyenv init - --no-rehash)" + _FILES="$(find salt/ -name "*.py" -exec git diff --name-only "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME" {} +)" + _FILES="$_FILES $(git diff --name-only "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME" setup.py)" + if [[ -z ${_FILES} ]]; then + echo "No pylint run, no changes found in the files" + echo "empty" pylint-reports.xml + else + tox -e pylint-salt ${_FILES} | tee pylint-report.xml + fi + ''' archiveArtifacts artifacts: 'pylint-report.xml' } } stage('test linting') { steps { - sh 'eval "$(pyenv init - --no-rehash)"; tox -e pylint-tests $(find tests/ -name "*.py" -exec git diff --name-only "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME" {} +) | tee pylint-report-tests.xml' + sh ''' + eval "$(pyenv init - --no-rehash)" + _FILES="$(find tests/ -name "*.py" -exec git diff --name-only "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME" setup.py {} +)" + if [[ -z ${_FILES} ]]; then + echo "No pylint run, no changes found in the files" + touch pylint-report-tests.xml + else + tox -e pylint-tests ${_FILES} | tee pylint-report-tests.xml + fi + ''' archiveArtifacts artifacts: 'pylint-report-tests.xml' } } From d1b7fb449c479b22eab17e32196bd4b7f1b017b9 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 28 Aug 2018 11:17:32 -0500 Subject: [PATCH 04/26] Revert "Update documentation to correctly state enable_gpu_grains default" This reverts commit 2fe675cfdd86ba370f3eec70dd50d149a70d04e5. --- conf/master | 2 +- conf/suse/master | 2 +- doc/man/salt.7 | 4 ++-- doc/ref/configuration/master.rst | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conf/master b/conf/master index f235abc072b..e88aa578c7c 100644 --- a/conf/master +++ b/conf/master @@ -125,7 +125,7 @@ # The master can take a while to start up when lspci and/or dmidecode is used # to populate the grains for the master. Enable if you want to see GPU hardware # data for your master. -# enable_gpu_grains: True +# enable_gpu_grains: False # The master maintains a job cache. While this is a great addition, it can be # a burden on the master for larger deployments (over 5000 minions). diff --git a/conf/suse/master b/conf/suse/master index 5f7f16fb4d9..e0a51c000fb 100644 --- a/conf/suse/master +++ b/conf/suse/master @@ -127,7 +127,7 @@ syndic_user: salt # The master can take a while to start up when lspci and/or dmidecode is used # to populate the grains for the master. Enable if you want to see GPU hardware # data for your master. -# enable_gpu_grains: True +# enable_gpu_grains: False # The master maintains a job cache. While this is a great addition, it can be # a burden on the master for larger deployments (over 5000 minions). diff --git a/doc/man/salt.7 b/doc/man/salt.7 index 4703af96e30..3dedd13dae2 100644 --- a/doc/man/salt.7 +++ b/doc/man/salt.7 @@ -5261,7 +5261,7 @@ sock_dir: /var/run/salt/master .UNINDENT .SS \fBenable_gpu_grains\fP .sp -Default: \fBFalse\fP +Default: \fBTrue\fP .sp Enable GPU hardware data for your master. Be aware that the master can take a while to start up when lspci and/or dmidecode is used to populate the @@ -14347,7 +14347,7 @@ and \fBmine_functions\fP\&. # The master can take a while to start up when lspci and/or dmidecode is used # to populate the grains for the master. Enable if you want to see GPU hardware # data for your master. -# enable_gpu_grains: True +# enable_gpu_grains: False # The master maintains a job cache. While this is a great addition, it can be # a burden on the master for larger deployments (over 5000 minions). diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 3829b963938..5105f954ecc 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -440,7 +440,7 @@ communication. ``enable_gpu_grains`` --------------------- -Default: ``False`` +Default: ``True`` Enable GPU hardware data for your master. Be aware that the master can take a while to start up when lspci and/or dmidecode is used to populate the From 49ed156dd5593a23fb2e337eecf998b5426fc1ef Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 28 Aug 2018 11:28:03 -0500 Subject: [PATCH 05/26] Add enable_gpu_grains to the minion config stub --- conf/minion | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/minion b/conf/minion index 7420713f96a..2a7cf6f98be 100644 --- a/conf/minion +++ b/conf/minion @@ -148,6 +148,11 @@ # Set the directory used to hold unix sockets. #sock_dir: /var/run/salt/minion +# The minion can take a while to start up when lspci and/or dmidecode is used +# to populate the grains for the minion. Set this to False if you do not need +# GPU hardware grains for your minion. +# enable_gpu_grains: True + # Set the default outputter used by the salt-call command. The default is # "nested". #output: nested From 638210a6da7f21cd418389990aef434d0c1c8a5b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 28 Aug 2018 11:28:27 -0500 Subject: [PATCH 06/26] Add validation type and default minion value for enable_gpu_grains --- salt/config/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 7c8b75bcdfd..a7e58bd5fa8 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -414,6 +414,9 @@ VALID_OPTS = { # Tell the loader to attempt to import *.pyx cython files if cython is available 'cython_enable': bool, + # Whether or not to load grains for the GPU + 'enable_gpu_grains': bool, + # Tell the loader to attempt to import *.zip archives 'enable_zip_modules': bool, @@ -1264,6 +1267,7 @@ DEFAULT_MINION_OPTS = { 'test': False, 'ext_job_cache': '', 'cython_enable': False, + 'enable_gpu_grains': True, 'enable_zip_modules': False, 'state_verbose': True, 'state_output': 'full', From ea1b53cb6f9b584f8d14676a747f620027278014 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 28 Aug 2018 11:39:53 -0500 Subject: [PATCH 07/26] Fix incorrect master docs for enable_gpu_grains --- doc/ref/configuration/master.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 5105f954ecc..d57205a2fcd 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -440,12 +440,16 @@ communication. ``enable_gpu_grains`` --------------------- -Default: ``True`` +Default: ``False`` Enable GPU hardware data for your master. Be aware that the master can take a while to start up when lspci and/or dmidecode is used to populate the grains for the master. +.. code-block:: yaml + + enable_gpu_grains: True + .. conf_master:: job_cache ``job_cache`` From 7372e9d6e3cb3cb647d7d8d0cfe8228b3765706b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 28 Aug 2018 11:40:14 -0500 Subject: [PATCH 08/26] Add minion documentation for enable_gpu_grains --- doc/ref/configuration/minion.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index ef8564105df..cb6b4e470f2 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -707,6 +707,22 @@ The directory where Unix sockets will be kept. sock_dir: /var/run/salt/minion +.. conf_minion:: enable_gpu_grains + +``enable_gpu_grains`` +--------------------- + +Default: ``True`` + +Enable GPU hardware data for your master. Be aware that the minion can +take a while to start up when lspci and/or dmidecode is used to populate the +grains for the minion, so this can be set to ``False`` if you do not need these +grains. + +.. code-block:: yaml + + enable_gpu_grains: False + .. conf_minion:: outputter_dirs ``outputter_dirs`` From e4264592602de2d7c13ce8595019e9d934c1b9b3 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 28 Aug 2018 12:06:16 -0600 Subject: [PATCH 09/26] Use a set to avoid duplicates --- salt/modules/mac_user.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/modules/mac_user.py b/salt/modules/mac_user.py index 72a443264c2..927e49ba4b1 100644 --- a/salt/modules/mac_user.py +++ b/salt/modules/mac_user.py @@ -467,7 +467,8 @@ def list_users(): salt '*' user.list_users ''' - return sorted([user.pw_name for user in pwd.getpwall()]) + # Use a set to not allow duplicates + return sorted(set(user.pw_name for user in pwd.getpwall())) def rename(name, new_name): From 56ea4eebcdfe8e8003748b15299a057d82b5fd80 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 28 Aug 2018 12:13:52 -0700 Subject: [PATCH 10/26] Account for more tests that are not in 2017.7 I also made sure there are no other missing tests since this bit me once already. --- tests/whitelist.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/whitelist.txt b/tests/whitelist.txt index 2ffd1917280..79c648af13a 100644 --- a/tests/whitelist.txt +++ b/tests/whitelist.txt @@ -9,7 +9,6 @@ integration.doc.test_man integration.grains.test_core integration.loader.test_ext_grains integration.loader.test_ext_modules -integration.logging.test_jid_logging integration.minion.test_blackout integration.minion.test_pillar integration.minion.test_timeout From 6cefbdf7b79dc6e6d7c63cf083e346f399b62ff4 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 28 Aug 2018 13:22:15 -0600 Subject: [PATCH 11/26] Add docs from a comment on issue 48758 --- salt/modules/win_lgpo.py | 120 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 119 insertions(+), 1 deletion(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 936f1cd983b..0fd368e5552 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -4896,7 +4896,7 @@ def _lookup_admin_template(policy_name, def get_policy_info(policy_name, policy_class, adml_language='en-US'): - ''' + r''' Returns information about a specified policy Args: @@ -4915,6 +4915,124 @@ def get_policy_info(policy_name, .. code-block:: bash salt '*' lgpo.get_policy_info 'Maximum password age' machine + + You can use ``lgpo.get_policy_info`` to get all the possible names that + could be used in a state file or from the command line (along with elements + that need to be set/etc). The key is to match the text you see in the + ``gpedit.msc`` gui exactly, including quotes around words or phrases. The + "full path" style is really only needed when there are multiple policies + that use the same base name. For example, ``Access data sources across + domains`` exists in ~10 different paths. If you put that through + ``get_policy_info`` you'll get back a message that it is used for multiple + policies and you need to be more specific. + + CLI Example: + + .. code-block:: bash + + salt-call --local lgpo.get_policy_info ShellRemoveOrderPrints_2 machine + + local: + ---------- + message: + policy_aliases: + - Turn off the "Order Prints" picture task + - ShellRemoveOrderPrints_2 + - System\Internet Communication Management\Internet Communication settings\Turn off the "Order Prints" picture task + policy_class: + machine + policy_elements: + policy_found: + True + policy_name: + ShellRemoveOrderPrints_2 + rights_assignment: + False + + Escaping can get tricky in cmd/Powershell. The following is an example of + escaping in Powershell using backquotes: + + .. code-block:: bash + + PS>salt-call --local lgpo.get_policy_info "Turn off the `\`"Order Prints`\`" picture task" machine + + local: + ---------- + message: + policy_aliases: + - Turn off the "Order Prints" picture task + - ShellRemoveOrderPrints_2 + - System\Internet Communication Management\Internet Communication settings\Turn off the "Order Prints" picture task + policy_class: + machine + policy_elements: + policy_found: + True + policy_name: + Turn off the "Order Prints" picture task + rights_assignment: + False + + This function can then be used to get the options available for specifying + Group Policy Objects to be used in state files. Based on the above any of + these *should* be usable: + + .. code-block:: bash + + internet_communications_settings: + lgpo.set: + - computer_policy: + Turn off the "Order Prints" picture task: Enabled + + .. code-block:: bash + + internet_communications_settings: + lgpo.set: + - computer_policy: + ShellRemoveOrderPrints_2: Enabled + + When using the full path, it might be a good idea to use single quotes + around the path: + + .. code-block:: bash + + internet_communications_settings: + lgpo.set: + - computer_policy: + 'System\Internet Communication Management\Internet Communication settings\Turn off the "Order Prints" picture task': 'Enabled' + + If you struggle to find the policy from ``get_policy_info`` using the name + as you see in ``gpedit.msc``, the names such as "ShellRemoveOrderPrints_2" + come from the ``.admx`` files. If you know nothing about ``.admx/.adml`` + relationships (ADML holds what you see in the GUI, ADMX holds the more + technical details), then this may be a little bit too much info, but here is + an example with the above policy using Powershell: + + + .. code-block:: bash + + PS>Get-ChildItem -Path C:\Windows\PolicyDefinitions -Recurse -Filter *.adml | Select-String "Order Prints" + + C:\windows\PolicyDefinitions\en-US\ICM.adml:152: Turn off the "Order Prints" picture task + C:\windows\PolicyDefinitions\en-US\ICM.adml:153: This policy setting specifies whether the "Order Prints Online" task is available from Picture Tasks in Windows folders. + C:\windows\PolicyDefinitions\en-US\ICM.adml:155:The Order Prints Online Wizard is used to download a list of providers and allow users to order prints online. + C:\windows\PolicyDefinitions\en-US\ICM.adml:157:If you enable this policy setting, the task "Order Prints Online" is removed from Picture Tasks in File Explorer folders. + + From this grep, we can see id "ShellRemoveOrderPrints" is the ID of the + string used to describe this policy, then we search for it in the ADMX: + + .. code-block:: bash + + PS>Get-ChildItem -Path C:\Windows\PolicyDefinitions -Recurse -Filter *.admx | Select-String "ShellRemoveOrderPrints" + + C:\windows\PolicyDefinitions\ICM.admx:661: + C:\windows\PolicyDefinitions\ICM.admx:671: + + Now we have two to pick from. And if you notice the ``class="Machine"`` and + ``class="User"`` (which details if it is a computer policy or user policy + respectively) the ``ShellRemoveOrderPrints_2`` is the "short name" we could + use to pass through ``get_policy_info`` to see what the module itself is + expecting. ''' # return the possible policy names and element names ret = {'policy_name': policy_name, From f9d7dbdb26ef18fe1980fb000938499cf5697c0c Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 28 Aug 2018 14:22:05 -0700 Subject: [PATCH 12/26] The autoruns module has not been renamed --- tests/whitelist.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/whitelist.txt b/tests/whitelist.txt index 79c648af13a..2631ebccd60 100644 --- a/tests/whitelist.txt +++ b/tests/whitelist.txt @@ -40,7 +40,7 @@ integration.modules.test_sysmod integration.modules.test_system integration.modules.test_test integration.modules.test_useradd -integration.modules.test_win_autoruns +integration.modules.test_autoruns integration.modules.test_win_dns_client integration.modules.test_win_pkg integration.reactor.test_reactor From 40d3f2ed83778f52f025d37da4f822910b99edb7 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 28 Aug 2018 15:33:49 -0600 Subject: [PATCH 13/26] Use dscl to get list of users --- salt/modules/mac_user.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/mac_user.py b/salt/modules/mac_user.py index 927e49ba4b1..5da993e764b 100644 --- a/salt/modules/mac_user.py +++ b/salt/modules/mac_user.py @@ -467,8 +467,8 @@ def list_users(): salt '*' user.list_users ''' - # Use a set to not allow duplicates - return sorted(set(user.pw_name for user in pwd.getpwall())) + users = _dscl(['/users'], 'list')['stdout'] + return users.split() def rename(name, new_name): From 2bf7edaa61778a8ff2d1fb6b6b1e5b98481b452e Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 28 Aug 2018 17:30:34 -0600 Subject: [PATCH 14/26] Import GLOBAL_ONLY --- salt/modules/win_repo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/modules/win_repo.py b/salt/modules/win_repo.py index 128032e5716..d08b52d0ecf 100644 --- a/salt/modules/win_repo.py +++ b/salt/modules/win_repo.py @@ -26,7 +26,8 @@ from salt.runners.winrepo import ( genrepo as _genrepo, update_git_repos as _update_git_repos, PER_REMOTE_OVERRIDES, - PER_REMOTE_ONLY + PER_REMOTE_ONLY, + GLOBAL_ONLY ) from salt.ext import six try: From 0ace5c137eb785780251ae8ca8c91175fc5ab8fa Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 28 Aug 2018 21:09:10 -0500 Subject: [PATCH 15/26] make file envs compatible with transport format --- salt/fileserver/__init__.py | 6 ++++++ salt/master.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/fileserver/__init__.py b/salt/fileserver/__init__.py index cbdf99d056d..ec561f0c03f 100644 --- a/salt/fileserver/__init__.py +++ b/salt/fileserver/__init__.py @@ -490,6 +490,12 @@ class Fileserver(object): return ret return list(ret) + def file_envs(self, load): + ''' + Return environments for all backends for requests from fileclient + ''' + return self.envs() + def init(self, back=None): ''' Initialize the backend, only do so if the fs supports an init function diff --git a/salt/master.py b/salt/master.py index c31a56327d3..8a103cccfd8 100644 --- a/salt/master.py +++ b/salt/master.py @@ -963,7 +963,7 @@ class AESFuncs(object): self._file_list_emptydirs = self.fs_.file_list_emptydirs self._dir_list = self.fs_.dir_list self._symlink_list = self.fs_.symlink_list - self._file_envs = self.fs_.envs + self._file_envs = self.fs_.file_envs def __verify_minion(self, id_, token): ''' From be7c041d75617a9cbdb5a11df7b8bce007d35de2 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 28 Aug 2018 21:14:05 -0500 Subject: [PATCH 16/26] add test --- salt/modules/cp.py | 7 +++++++ tests/integration/modules/test_cp.py | 3 +++ 2 files changed, 10 insertions(+) diff --git a/salt/modules/cp.py b/salt/modules/cp.py index 58e344d6445..1a3bcb4d4bf 100644 --- a/salt/modules/cp.py +++ b/salt/modules/cp.py @@ -296,6 +296,13 @@ def get_file(path, gzip) +def envs(): + ''' + List available environments for fileserver + ''' + return _client().envs() + + def get_template(path, dest, template='jinja', diff --git a/tests/integration/modules/test_cp.py b/tests/integration/modules/test_cp.py index 57b9aae9885..28f68dc3527 100644 --- a/tests/integration/modules/test_cp.py +++ b/tests/integration/modules/test_cp.py @@ -614,3 +614,6 @@ class CPModuleTest(ModuleCase): self.assertTrue(os.path.isfile(tgt_cache_file), 'File was not cached on the master') finally: os.unlink(tgt_cache_file) + + def test_envs(self): + self.assertEqual(self.run_function('cp.envs'), ['base', 'prod']) From b332bebb2068582de94115f7332ef3aea9e18d78 Mon Sep 17 00:00:00 2001 From: Daniel A Wozniak Date: Wed, 29 Aug 2018 06:59:55 +0000 Subject: [PATCH 17/26] Fix archive tests for py3 --- tests/integration/modules/test_archive.py | 14 +++++++++----- tests/integration/modules/test_system.py | 3 ++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/integration/modules/test_archive.py b/tests/integration/modules/test_archive.py index 915a6b2077c..2ba62abb799 100644 --- a/tests/integration/modules/test_archive.py +++ b/tests/integration/modules/test_archive.py @@ -65,6 +65,10 @@ class ArchiveTest(ModuleCase): else: filename = 'file' with salt.utils.fopen(os.path.join(self.src, filename), 'wb') as theorem: + if six.PY3 and salt.utils.is_windows(): + encoding = 'utf-8' + else: + encoding = None theorem.write(salt.utils.to_bytes(textwrap.dedent('''\ Compression theorem of computational complexity theory: @@ -82,7 +86,7 @@ class ArchiveTest(ModuleCase): and $\\mathrm C(φ_i) ⊊ \\mathrm{C}(φ_{f(i)})$. - '''))) + '''), encoding=encoding)) # Create destination os.makedirs(self.dst) @@ -124,10 +128,10 @@ class ArchiveTest(ModuleCase): dir_in_ret = None file_in_ret = None for line in ret: - if normdir(self.src) in line \ - and not normdir(self.src_file) in line: + if normdir(self.src) in os.path.normcase(line) \ + and not normdir(self.src_file) in os.path.normcase(line): dir_in_ret = True - if normdir(self.src_file) in line: + if normdir(self.src_file) in os.path.normcase(line): file_in_ret = True # Assert number of lines, reporting of source directory and file @@ -250,7 +254,7 @@ class ArchiveTest(ModuleCase): # Test create archive ret = self.run_function('archive.unzip', [self.arch, self.dst]) self.assertTrue(isinstance(ret, list), six.text_type(ret)) - self._assert_artifacts_in_ret(ret, unix_sep=True if six.PY2 else False) + self._assert_artifacts_in_ret(ret, unix_sep=False) self._tear_down() diff --git a/tests/integration/modules/test_system.py b/tests/integration/modules/test_system.py index f41b4e6b560..7497ffedf8d 100644 --- a/tests/integration/modules/test_system.py +++ b/tests/integration/modules/test_system.py @@ -11,7 +11,7 @@ import subprocess # Import Salt Testing libs from tests.support.case import ModuleCase from tests.support.unit import skipIf -from tests.support.helpers import destructiveTest, skip_if_not_root +from tests.support.helpers import destructiveTest, skip_if_not_root, flaky # Import salt libs import salt.utils @@ -251,6 +251,7 @@ class SystemModuleTest(ModuleCase): self.assertTrue(self._same_times(time_now, cmp_time), msg=msg) self._test_hwclock_sync() + @flaky @destructiveTest @skip_if_not_root def test_set_system_time(self): From 251f321135bfb955b68d54f49f94185220bc3901 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 29 Aug 2018 07:50:13 -0500 Subject: [PATCH 18/26] pass load on --- salt/fileserver/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/fileserver/__init__.py b/salt/fileserver/__init__.py index ec561f0c03f..94708a79d43 100644 --- a/salt/fileserver/__init__.py +++ b/salt/fileserver/__init__.py @@ -490,11 +490,13 @@ class Fileserver(object): return ret return list(ret) - def file_envs(self, load): + def file_envs(self, load=None): ''' Return environments for all backends for requests from fileclient ''' - return self.envs() + if load is None: + load = {} + return self.envs(**load) def init(self, back=None): ''' From bdf3df3397bd4ea5348328cbe9ffc68f5317e4e9 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 29 Aug 2018 08:46:11 -0500 Subject: [PATCH 19/26] add cli example --- salt/modules/cp.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/modules/cp.py b/salt/modules/cp.py index 1a3bcb4d4bf..c91bcd4eba7 100644 --- a/salt/modules/cp.py +++ b/salt/modules/cp.py @@ -299,6 +299,12 @@ def get_file(path, def envs(): ''' List available environments for fileserver + + CLI Example + + .. code-block:: bash + + salt '*' cp.envs ''' return _client().envs() From 7dec9fca0c3fdae2ec0de4757fe2a549e5393635 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 24 Aug 2018 17:07:16 -0400 Subject: [PATCH 20/26] Add flaky decorator to serializer test Fixes https://github.com/saltstack/salt-jenkins/issues/1093 --- tests/unit/serializers/test_serializers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/serializers/test_serializers.py b/tests/unit/serializers/test_serializers.py index 2520a462d86..1b4b842732f 100644 --- a/tests/unit/serializers/test_serializers.py +++ b/tests/unit/serializers/test_serializers.py @@ -24,7 +24,7 @@ from salt.utils.odict import OrderedDict # Import test support libs from tests.support.helpers import flaky -SKIP_MESSAGE = '%s is unavailable, do prerequisites have been met?' +SKIP_MESSAGE = '%s is unavailable, have prerequisites been met?' @flaky(condition=six.PY3) @@ -93,6 +93,7 @@ class TestSerializers(TestCase): @skipIf(not yaml.available, SKIP_MESSAGE % 'yaml') @skipIf(not yamlex.available, SKIP_MESSAGE % 'sls') + @flaky def test_compare_sls_vs_yaml_with_jinja(self): tpl = '{{ data }}' env = jinja2.Environment() From 6c01662665fb67503d7fac83272449055d656e94 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 29 Aug 2018 13:59:00 -0500 Subject: [PATCH 21/26] remove cmd key from load --- salt/fileserver/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/fileserver/__init__.py b/salt/fileserver/__init__.py index 94708a79d43..18f0c845e5e 100644 --- a/salt/fileserver/__init__.py +++ b/salt/fileserver/__init__.py @@ -496,6 +496,7 @@ class Fileserver(object): ''' if load is None: load = {} + load.pop('cmd', None) return self.envs(**load) def init(self, back=None): From d02ec34790a485eadf9610f3fbae6db8ffe2e75b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 29 Aug 2018 14:13:07 -0500 Subject: [PATCH 22/26] Allow our custom yaml dumper to NamespacedDictWrapper objects This special dict class did not have a representer assigned to it, so PyYAML would raise an exception when attempting to dump it. This also removes the name overriding, as its implementation was ill-conceived and will be deprecated in a later branch (it's only used in this one place so deprecating should be of no-to-minimal impact). --- salt/loader.py | 4 ++-- salt/utils/yamldumper.py | 9 +++++++++ tests/integration/output/test_output.py | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/salt/loader.py b/salt/loader.py index e8f0bfaef83..b4e7dbb70c0 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -1357,11 +1357,11 @@ class LazyLoader(salt.utils.lazy.LazyDict): ''' if '__grains__' not in self.pack: self.context_dict['grains'] = opts.get('grains', {}) - self.pack['__grains__'] = salt.utils.context.NamespacedDictWrapper(self.context_dict, 'grains', override_name='grains') + self.pack['__grains__'] = salt.utils.context.NamespacedDictWrapper(self.context_dict, 'grains') if '__pillar__' not in self.pack: self.context_dict['pillar'] = opts.get('pillar', {}) - self.pack['__pillar__'] = salt.utils.context.NamespacedDictWrapper(self.context_dict, 'pillar', override_name='pillar') + self.pack['__pillar__'] = salt.utils.context.NamespacedDictWrapper(self.context_dict, 'pillar') mod_opts = {} for key, val in list(opts.items()): diff --git a/salt/utils/yamldumper.py b/salt/utils/yamldumper.py index 6205cb68030..c8d221449ae 100644 --- a/salt/utils/yamldumper.py +++ b/salt/utils/yamldumper.py @@ -17,6 +17,7 @@ except ImportError: import yaml import collections +import salt.utils.context from salt.utils.odict import OrderedDict try: @@ -54,6 +55,14 @@ SafeOrderedDumper.add_representer( collections.defaultdict, yaml.representer.SafeRepresenter.represent_dict ) +OrderedDumper.add_representer( + salt.utils.context.NamespacedDictWrapper, + yaml.representer.SafeRepresenter.represent_dict +) +SafeOrderedDumper.add_representer( + salt.utils.context.NamespacedDictWrapper, + yaml.representer.SafeRepresenter.represent_dict +) if HAS_IOFLO: OrderedDumper.add_representer(odict, represent_ordereddict) diff --git a/tests/integration/output/test_output.py b/tests/integration/output/test_output.py index 78e38f79133..17222963f3a 100644 --- a/tests/integration/output/test_output.py +++ b/tests/integration/output/test_output.py @@ -7,6 +7,7 @@ from __future__ import absolute_import import os import traceback +import yaml # Import Salt Testing Libs from tests.support.case import ShellCase @@ -81,6 +82,20 @@ class OutputReturnTest(ShellCase): ret = self.run_call('test.ping --out=yaml') self.assertEqual(ret, expected) + def test_output_yaml_namespaced_dict_wrapper(self): + ''' + Tests the ability to dump a NamespacedDictWrapper instance, as used in + magic dunders like __grains__ and __pillar__ + + See https://github.com/saltstack/salt/issues/49269 + ''' + dumped_yaml = '\n'.join(self.run_call('grains.items --out=yaml')) + loaded_yaml = yaml.load(dumped_yaml) + # We just want to check that the dumped YAML loades as a dict with a + # single top-level key, we don't care about the real contents. + assert isinstance(loaded_yaml, dict) + assert list(loaded_yaml) == ['local'] + def test_output_unicodebad(self): ''' Tests outputter reliability with utf8 From d74fab7703e3d60295302d625b150b2847b49274 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 29 Aug 2018 14:45:17 -0700 Subject: [PATCH 23/26] Clean up exception handling on py3 --- salt/modules/win_system.py | 8 ++++---- salt/modules/win_useradd.py | 6 +----- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/salt/modules/win_system.py b/salt/modules/win_system.py index 8ff79d86030..4500954e8db 100644 --- a/salt/modules/win_system.py +++ b/salt/modules/win_system.py @@ -303,7 +303,7 @@ def shutdown(message=None, timeout=5, force_close=True, reboot=False, # pylint: force_close, reboot) return True except pywintypes.error as exc: - (number, context, message) = exc + (number, context, message) = exc.args log.error('Failed to shutdown the system') log.error('nbr: {0}'.format(number)) log.error('ctx: {0}'.format(context)) @@ -346,7 +346,7 @@ def shutdown_abort(): win32api.AbortSystemShutdown('127.0.0.1') return True except pywintypes.error as exc: - (number, context, message) = exc + (number, context, message) = exc.args log.error('Failed to abort system shutdown') log.error('nbr: {0}'.format(number)) log.error('ctx: {0}'.format(context)) @@ -485,7 +485,7 @@ def set_computer_desc(desc=None): try: win32net.NetServerSetInfo(None, 101, system_info) except win32net.error as exc: - (number, context, message) = exc + (number, context, message) = exc.args log.error('Failed to update system') log.error('nbr: {0}'.format(number)) log.error('ctx: {0}'.format(context)) @@ -1016,7 +1016,7 @@ def set_system_date_time(years=None, try: date_time = win32api.GetLocalTime() except win32api.error as exc: - (number, context, message) = exc + (number, context, message) = exc.args log.error('Failed to get local time') log.error('nbr: {0}'.format(number)) log.error('ctx: {0}'.format(context)) diff --git a/salt/modules/win_useradd.py b/salt/modules/win_useradd.py index 50aebade96c..0c2017a3b24 100644 --- a/salt/modules/win_useradd.py +++ b/salt/modules/win_useradd.py @@ -164,7 +164,6 @@ def add(name, try: win32net.NetUserAdd(None, 1, user_info) except win32net.error as exc: - (number, context, message) = exc log.error('Failed to create user {0}'.format(name)) log.error('nbr: {0}'.format(exc.winerror)) log.error('ctx: {0}'.format(exc.funcname)) @@ -266,7 +265,6 @@ def update(name, try: user_info = win32net.NetUserGetInfo(None, name, 4) except win32net.error as exc: - (number, context, message) = exc log.error('Failed to update user {0}'.format(name)) log.error('nbr: {0}'.format(exc.winerror)) log.error('ctx: {0}'.format(exc.funcname)) @@ -326,7 +324,6 @@ def update(name, try: win32net.NetUserSetInfo(None, name, 4, user_info) except win32net.error as exc: - (number, context, message) = exc log.error('Failed to update user {0}'.format(name)) log.error('nbr: {0}'.format(exc.winerror)) log.error('ctx: {0}'.format(exc.funcname)) @@ -415,7 +412,7 @@ def delete(name, sid = getUserSid(name) win32profile.DeleteProfile(sid) except pywintypes.error as exc: - (number, context, message) = exc + (number, context, message) = exc.args if number == 2: # Profile Folder Not Found pass else: @@ -429,7 +426,6 @@ def delete(name, try: win32net.NetUserDel(None, name) except win32net.error as exc: - (number, context, message) = exc log.error('Failed to delete user {0}'.format(name)) log.error('nbr: {0}'.format(exc.winerror)) log.error('ctx: {0}'.format(exc.funcname)) From 9daa992c6ee3e4074620ded602f92dd25f8693a7 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 29 Aug 2018 15:22:41 -0700 Subject: [PATCH 24/26] Fix group remove test logic --- tests/integration/modules/test_useradd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/modules/test_useradd.py b/tests/integration/modules/test_useradd.py index ab9927ef3d3..00709ac9c85 100644 --- a/tests/integration/modules/test_useradd.py +++ b/tests/integration/modules/test_useradd.py @@ -268,8 +268,8 @@ class UseraddModuleTestWindows(ModuleCase): self._add_group() self.run_function('user.addgroup', [self.user_name, self.group_name]) self.assertIn(self.group_name, self.run_function('user.list_groups', [self.user_name])) - self.run_function('user.removegroup', [self.group_name]) - self.assertIn(self.group_name, self.run_function('user.list_groups', [self.user_name])) + self.run_function('user.removegroup', [self.user_name, self.group_name]) + self.assertNotIn(self.group_name, self.run_function('user.list_groups', [self.user_name])) def test_user_rename(self): ''' From 535d83e39e8d6acb5c95d781df45aacc65b3e3c7 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 29 Aug 2018 16:39:10 -0600 Subject: [PATCH 25/26] Fix test for list_users --- tests/unit/modules/test_mac_user.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/unit/modules/test_mac_user.py b/tests/unit/modules/test_mac_user.py index 6581f6c252d..bb5f63e4f0f 100644 --- a/tests/unit/modules/test_mac_user.py +++ b/tests/unit/modules/test_mac_user.py @@ -312,6 +312,11 @@ class MacUserTestCase(TestCase, LoaderModuleMockMixin): ''' Tests the list of all users ''' - with patch('pwd.getpwall', MagicMock(return_value=self.mock_pwall)): - ret = ['_amavisd', '_appleevents', '_appowner'] - self.assertEqual(mac_user.list_users(), ret) + expected = ['spongebob', 'patrick', 'squidward'] + mock_run = MagicMock(return_value={'pid': 4948, + 'retcode': 0, + 'stderr': '', + 'stdout': '\n'.join(expected)}) + with patch.dict(mac_user.__grains__, {'osrelease_info': (10, 9, 1)}), \ + patch.dict(mac_user.__salt__, {'cmd.run_all': mock_run}): + self.assertEqual(mac_user.list_users(), expected) From fb97b00130ac0f3c305a656ba0010b59a6d057e9 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 29 Aug 2018 16:40:56 -0600 Subject: [PATCH 26/26] Fix broken path to libsodium tarball --- pkg/osx/build_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/osx/build_env.sh b/pkg/osx/build_env.sh index 5d7f1bac5a3..b6542844e9c 100755 --- a/pkg/osx/build_env.sh +++ b/pkg/osx/build_env.sh @@ -161,7 +161,7 @@ sudo -H $MAKE install ############################################################################ echo -n -e "\033]0;Build_Env: libsodium\007" -PKGURL="https://download.libsodium.org/libsodium/releases/libsodium-1.0.13.tar.gz" +PKGURL="https://download.libsodium.org/libsodium/releases/old/libsodium-1.0.13.tar.gz" PKGDIR="libsodium-1.0.13" download $PKGURL