Merge branch '2017.7' into 'develop'

Conflicts:
  - salt/pillar/git_pillar.py
  - salt/utils/files.py
  - salt/utils/process.py
  - tests/unit/utils/test_which.py

conf/master

@@ -59,15 +59,14 @@
 
 # Directory for custom modules. This directory can contain subdirectories for
 # each of Salt's module types such as "runners", "output", "wheel", "modules",
-# "states", "returners", etc.
-#extension_modules: <no default>
+# "states", "returners", "engines", "utils", etc.
+#extension_modules: /var/cache/salt/master/extmods
 
 # Directory for custom modules. This directory can contain subdirectories for
 # each of Salt's module types such as "runners", "output", "wheel", "modules",
-# "states", "returners", "engines", etc.
+# "states", "returners", "engines", "utils", etc.
 # Like 'extension_modules' but can take an array of paths
-#module_dirs: <no default>
-#   - /var/cache/salt/minion/extmods
+#module_dirs: []
 
 # Verify and set permissions on configuration directories at startup:
 #verify_env: True

doc/ref/configuration/master.rst

@@ -183,8 +183,8 @@ The directory to store the pki authentication keys.
 
 Directory for custom modules. This directory can contain subdirectories for
 each of Salt's module types such as ``runners``, ``output``, ``wheel``,
-``modules``, ``states``, ``returners``, ``engines``, etc. This path is appended to
-:conf_master:`root_dir`.
+``modules``, ``states``, ``returners``, ``engines``, ``utils``, etc.
+This path is appended to :conf_master:`root_dir`.
 
 .. code-block:: yaml
 

doc/topics/utils/index.rst

@@ -91,8 +91,8 @@ Also you could even write your utility modules in object oriented fashion:
 
     # -*- coding: utf-8 -*-
     '''
-    My utils module
-    ---------------
+    My OOP-style utils module
+    -------------------------
 
     This module contains common functions for use in my other custom types.
     '''

salt/grains/core.py

@@ -805,6 +805,8 @@ def _virtual(osdata):
                         grains['virtual_subtype'] = 'ovirt'
                     elif 'Google' in output:
                         grains['virtual'] = 'gce'
+                    elif 'BHYVE' in output:
+                        grains['virtual'] = 'bhyve'
             except IOError:
                 pass
     elif osdata['kernel'] == 'FreeBSD':

salt/modules/dockermod.py

@@ -3885,8 +3885,9 @@ def save(name,
         saved_path = salt.utils.files.mkstemp()
     else:
         saved_path = path
-
-    cmd = ['docker', 'save', '-o', saved_path, inspect_image(name)['Id']]
+    # use the image name if its valid if not use the image id
+    image_to_save = name if name in inspect_image(name)['RepoTags'] else inspect_image(name)['Id']
+    cmd = ['docker', 'save', '-o', saved_path, image_to_save]
     time_started = time.time()
     result = __salt__['cmd.run_all'](cmd, python_shell=False)
     if result['retcode'] != 0:

salt/modules/win_lgpo.py

@@ -4025,14 +4025,14 @@ def _write_regpol_data(data_to_write,
         reg_pol_header = u'\u5250\u6765\x01\x00'
         if not os.path.exists(policy_file_path):
             ret = __salt__['file.makedirs'](policy_file_path)
-        with salt.utils.files.fopen(policy_file_path, 'wb') as pol_file:
+        with salt.utils.fopen(policy_file_path, 'wb') as pol_file:
             if not data_to_write.startswith(reg_pol_header):
                 pol_file.write(reg_pol_header.encode('utf-16-le'))
             pol_file.write(data_to_write.encode('utf-16-le'))
         try:
             gpt_ini_data = ''
             if os.path.exists(gpt_ini_path):
-                with salt.utils.files.fopen(gpt_ini_path, 'rb') as gpt_file:
+                with salt.utils.fopen(gpt_ini_path, 'rb') as gpt_file:
                     gpt_ini_data = gpt_file.read()
             if not _regexSearchRegPolData(r'\[General\]\r\n', gpt_ini_data):
                 gpt_ini_data = '[General]\r\n' + gpt_ini_data
@@ -4087,7 +4087,7 @@ def _write_regpol_data(data_to_write,
                         int("{0}{1}".format(str(version_nums[0]).zfill(4), str(version_nums[1]).zfill(4)), 16),
                         gpt_ini_data[general_location.end():])
             if gpt_ini_data:
-                with salt.utils.files.fopen(gpt_ini_path, 'wb') as gpt_file:
+                with salt.utils.fopen(gpt_ini_path, 'wb') as gpt_file:
                     gpt_file.write(gpt_ini_data)
         except Exception as e:
             msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format(

salt/utils/files.py

@@ -7,11 +7,13 @@ import contextlib
 import errno
 import logging
 import os
+import re
 import shutil
 import stat
 import subprocess
 import tempfile
 import time
+import urllib
 
 # Import Salt libs
 import salt.utils  # Can be removed when backup_minion is moved
@@ -460,3 +462,39 @@ def is_fcntl_available(check_sunos=False):
     if check_sunos and salt.utils.platform.is_sunos():
         return False
     return HAS_FCNTL
+
+
+def safe_filename_leaf(file_basename):
+    '''
+    Input the basename of a file, without the directory tree, and returns a safe name to use
+    i.e. only the required characters are converted by urllib.quote
+    If the input is a PY2 String, output a PY2 String. If input is Unicode output Unicode.
+    For consistency all platforms are treated the same. Hard coded to utf8 as its ascii compatible
+    windows is \\ / : * ? " < > | posix is /
+
+    .. versionadded:: 2017.7.2
+    '''
+    def _replace(re_obj):
+        return urllib.quote(re_obj.group(0), safe=u'')
+    if not isinstance(file_basename, six.text_type):
+        # the following string is not prefixed with u
+        return re.sub('[\\\\:/*?"<>|]',
+                      _replace,
+                      six.text_type(file_basename, 'utf8').encode('ascii', 'backslashreplace'))
+    # the following string is prefixed with u
+    return re.sub(u'[\\\\:/*?"<>|]', _replace, file_basename, flags=re.UNICODE)
+
+
+def safe_filepath(file_path_name):
+    '''
+    Input the full path and filename, splits on directory separator and calls safe_filename_leaf for
+    each part of the path.
+
+    .. versionadded:: 2017.7.2
+    '''
+    (drive, path) = os.path.splitdrive(file_path_name)
+    path = os.sep.join([safe_filename_leaf(file_section) for file_section in file_path_name.rsplit(os.sep)])
+    if drive:
+        return os.sep.join([drive, path])
+    else:
+        return path

salt/utils/process.py

@@ -15,6 +15,7 @@ import contextlib
 import subprocess
 import multiprocessing
 import multiprocessing.util
+import socket
 
 
 # Import salt libs
@@ -59,7 +60,20 @@ def notify_systemd():
     except ImportError:
         if salt.utils.path.which('systemd-notify') \
                 and systemd_notify_call('--booted'):
-            return systemd_notify_call('--ready')
+            # Notify systemd synchronously
+            notify_socket = os.getenv('NOTIFY_SOCKET')
+            if notify_socket:
+                # Handle abstract namespace socket
+                if notify_socket.startswith('@'):
+                    notify_socket = '\0{0}'.format(notify_socket[1:])
+                try:
+                    sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+                    sock.connect(notify_socket)
+                    sock.sendall('READY=1'.encode())
+                    sock.close()
+                except socket.error:
+                    return systemd_notify_call('--ready')
+                return True
         return False
 
     if systemd.daemon.booted():

tests/unit/utils/test_which.py

@@ -44,18 +44,21 @@ class TestWhich(TestCase):
                 # The second, iterating through $PATH, should also return False,
                 # still checking for Linux
                 False,
+                # We will now also return False once so we get a .EXE back from
+                # the function, see PATHEXT below.
+                False,
                 # Lastly return True, this is the windows check.
                 True
             ]
             # Let's patch os.environ to provide a custom PATH variable
-            with patch.dict(os.environ, {'PATH': '/bin'}):
+            with patch.dict(os.environ, {'PATH': '/bin',
+                                         'PATHEXT': '.COM;.EXE;.BAT;.CMD'}):
                 # Let's also patch is_windows to return True
                 with patch('salt.utils.platform.is_windows', lambda: True):
                     with patch('os.path.isfile', lambda x: True):
                         self.assertEqual(
                             salt.utils.path.which('this-binary-exists-under-windows'),
-                            # The returned path should return the .exe suffix
-                            '/bin/this-binary-exists-under-windows.EXE'
+                            os.path.join('/bin', 'this-binary-exists-under-windows.EXE')
                         )
 
     def test_missing_binary_in_windows(self):
@@ -106,6 +109,5 @@ class TestWhich(TestCase):
                     with patch('os.path.isfile', lambda x: True):
                         self.assertEqual(
                             salt.utils.path.which('this-binary-exists-under-windows'),
-                            # The returned path should return the .exe suffix
-                            '/bin/this-binary-exists-under-windows.CMD'
+                            os.path.join('/bin', 'this-binary-exists-under-windows.CMD')
                         )