Merge branch '2017.7' into '2018.3'

No conflicts.

.ci/lint

@@ -22,6 +22,13 @@ pipeline {
         stage('setup') {
             steps {
                 sh '''
+                # Need -M to detect renames otherwise they are reported as Delete and Add, need -C to detect copies, -C includes -M
+                # -M is on by default in git 2.9+
+                git diff --name-status -l99999 -C "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME" > file-list-status.log
+                # the -l increase the search limit, lets use awk so we do not need to repeat the search above.
+                gawk 'BEGIN {FS="\\t"} {if ($1 != "D") {print $NF}}' file-list-status.log > file-list-changed.log
+                gawk 'BEGIN {FS="\\t"} {if ($1 == "D") {print $NF}}' file-list-status.log > file-list-deleted.log
+                touch pylint-report-salt.log pylint-report-tests.log
                 eval "$(pyenv init -)"
                 pyenv --version
                 pyenv install --skip-existing 2.7.14
@@ -30,40 +37,38 @@ pipeline {
                 python --version
                 pip install tox
                 '''
+                archiveArtifacts artifacts: 'file-list-status.log,file-list-changed.log,file-list-deleted.log'
             }
         }
         stage('linting') {
             failFast false
             parallel {
                 stage('salt linting') {
+                    when {
+                        expression { return readFile('file-list-changed.log') =~ /(?i)(^|\n)(salt\/.*\.py|setup\.py)\n/ }
+                    }
                     steps {
                         sh '''
                         eval "$(pyenv init - --no-rehash)"
-                        _FILES="$(find salt/ -name "*.py" -exec git diff --name-only "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME" {} +)"
-                        _FILES="$_FILES $(git diff --name-only "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME" setup.py)"
-                        if [[ -z ${_FILES} ]]; then
-                            echo "No pylint run, no changes found in the files"
-                            echo "empty" pylint-reports.xml
-                        else
-                            tox -e pylint-salt ${_FILES} | tee pylint-report.xml
-                        fi
+                        grep -Ei '^salt/.*\\.py$|^setup\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' tox -e pylint-salt | tee pylint-report-salt.log
+                        # remove color escape coding
+                        sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-salt.log
                         '''
-                        archiveArtifacts artifacts: 'pylint-report.xml'
+                        archiveArtifacts artifacts: 'pylint-report-salt.log'
                     }
                 }
                 stage('test linting') {
+                    when {
+                        expression { return readFile('file-list-changed.log') =~ /(?i)(^|\n)tests\/.*\.py\n/ }
+                    }
                     steps {
                         sh '''
                         eval "$(pyenv init - --no-rehash)"
-                        _FILES="$(find tests/ -name "*.py" -exec git diff --name-only "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME" setup.py {} +)"
-                        if [[ -z ${_FILES} ]]; then
-                            echo "No pylint run, no changes found in the files"
-                            touch pylint-report-tests.xml
-                        else
-                            tox -e pylint-tests ${_FILES} | tee pylint-report-tests.xml
-                        fi
+                        grep -Ei '^tests/.*\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' tox -e pylint-tests | tee pylint-report-tests.log
+                        # remove color escape coding
+                        sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-tests.log
                         '''
-                        archiveArtifacts artifacts: 'pylint-report-tests.xml'
+                        archiveArtifacts artifacts: 'pylint-report-tests.log'
                     }
                 }
             }
@@ -74,7 +79,7 @@ pipeline {
             step([$class: 'WarningsPublisher',
                 parserConfigurations: [[
                     parserName: 'PyLint',
-                    pattern: 'pylint-report*.xml'
+                    pattern: 'pylint-report*.log'
                 ]],
                 failedTotalAll: '0',
                 useDeltaValues: false,

doc/topics/releases/2017.7.8.rst

@@ -1,9 +1,8 @@
-========================================
-In Progress: Salt 2017.7.8 Release Notes
-========================================
+===========================
+Salt 2017.7.8 Release Notes
+===========================
 
-Version 2017.7.8 is an **unreleased** bugfix release for :ref:`2017.7.0 <release-2017-7-0>`.
-This release is still in progress and has not been released yet.
+Version 2017.7.8 is a security and bugfix release for :ref:`2017.7.0 <release-2017-7-0>`.
 
 Statistics
 ==========
@@ -14,6 +13,12 @@ Statistics
 
 - Contributors: **52** (`AVeenstra`_, `Ch3LL`_, `Circuitsoft`_, `DmitryKuzmenko`_, `KaiSforza`_, `Martin819`_, `OrlandoArcapix`_, `UtahDave`_, `Vaelatern`_, `abednarik`_, `asnell`_, `b1naryth1ef`_, `baniobloom`_, `basepi`_, `bdrung`_, `beornf`_, `bmcorser`_, `bowmanjd-lms`_, `damon-atkins`_, `darkpixel`_, `discogestalt`_, `doesitblend`_, `dqminh`_, `dubb-b`_, `dwoz`_, `frankiexyz`_, `frogunder`_, `fzipi`_, `garethgreenaway`_, `grokrecursion`_, `gtmanfred`_, `jacksontj`_, `jagguli`_, `lejambon`_, `lomeroe`_, `lordcirth`_, `lusche`_, `mbunkus`_, `meaksh`_, `mirceaulinic`_, `nbraud`_, `pritambaral`_, `ralex`_, `rallytime`_, `rmcintosh`_, `slaws`_, `terminalmage`_, `twangboy`_, `twellspring`_, `wyardley`_, `xetix`_, `zer0def`_)
 
+Security Fix
+============
+
+CVE-2018-15751 Remote command execution and incorrect access control when using salt-api.
+
+CVE-2018-15750 Directory traversal vulnerability when using salt-api. Allows an attacker to determine what files exist on a server when querying /run or /events.
 
 New win_snmp behavior
 =====================

doc/topics/troubleshooting/yaml_idiosyncrasies.rst

@@ -77,11 +77,11 @@ deeply-nested dict can be declared with curly braces:
         - group: root
         - mode: 644
         - template: jinja
-        - context:
-          custom_var: "override"
-        - defaults:
-            custom_var: "default value"
-            other_var: 123
+        - context: {
+          custom_var: "override" }
+        - defaults: {
+          custom_var: "default value",
+          other_var: 123 }
 
 Here is a more concrete example of how YAML actually handles these
 indentations, using the Python interpreter on the command line:

doc/topics/tutorials/http.rst

@@ -74,15 +74,15 @@ be overridden with the ``method`` argument:
     salt.utils.http.query('http://example.com/delete/url', 'DELETE')
 
 When using the ``POST`` method (and others, such as ``PUT``), extra data is usually
-sent as well. This data can be sent directly, in whatever format is
-required by the remote server (XML, JSON, plain text, etc).
+sent as well. This data can be sent directly (would be URL encoded when necessary),
+or in whatever format is required by the remote server (XML, JSON, plain text, etc).
 
 .. code-block:: python
 
     salt.utils.http.query(
-        'http://example.com/delete/url',
+        'http://example.com/post/url',
         method='POST',
-        data=json.loads(mydict)
+        data=json.dumps(mydict)
     )
 
 Data Formatting and Templating

salt/netapi/rest_cherrypy/app.py

@@ -1174,6 +1174,13 @@ class LowDataAdapter(object):
                 except (TypeError, ValueError):
                     raise cherrypy.HTTPError(401, 'Invalid token')
 
+            if 'token' in chunk:
+                # Make sure that auth token is hex
+                try:
+                    int(chunk['token'], 16)
+                except (TypeError, ValueError):
+                    raise cherrypy.HTTPError(401, 'Invalid token')
+
             if client:
                 chunk['client'] = client
 

salt/spm/pkgfiles/local.py

@@ -133,7 +133,7 @@ def install_file(package, formula_tar, member, formula_def, conn=None):
             elif tag in ('s', 'm'):
                 pass
 
-    if new_name.startswith('{0}/_'.format(package)):
+    if member.name.startswith('{0}/_'.format(package)):
         if node_type in ('master', 'minion'):
             # Module files are distributed via extmods directory
             member.name = member.name.replace('{0}/_'.format(package), '')
@@ -145,7 +145,7 @@ def install_file(package, formula_tar, member, formula_def, conn=None):
         else:
             # Module files are distributed via _modules, _states, etc
             member.name = member.name.replace('{0}/'.format(package), '')
-    elif new_name == '{0}/pillar.example'.format(package):
+    elif member.name == '{0}/pillar.example'.format(package):
         # Pillars are automatically put in the pillar_path
         member.name = '{0}.sls.orig'.format(package)
         out_path = conn['pillar_path']

tests/support/helpers.py

@@ -1591,7 +1591,11 @@ def win32_kill_process_tree(pid, sig=signal.SIGTERM, include_parent=True,
     '''
     if pid == os.getpid():
         raise RuntimeError("I refuse to kill myself")
-    parent = psutil.Process(pid)
+    try:
+        parent = psutil.Process(pid)
+    except psutil.NoSuchProcess:
+        log.debug("PID not found alive: %d", pid)
+        return ([], [])
     children = parent.children(recursive=True)
     if include_parent:
         children.append(parent)