Merge branch '2017.7' into 'develop'

Conflicts:
  - salt/state.py
  - salt/utils/gitfs.py
This commit is contained in:
rallytime 2017-09-06 09:39:27 -04:00
commit 5701d54516
15 changed files with 165 additions and 57 deletions

60
.github/CODEOWNERS vendored Normal file
View file

@ -0,0 +1,60 @@
# SALTSTACK CODE OWNERS
# See https://help.github.com/articles/about-codeowners/
# for more info about CODEOWNERS file
# Lines starting with '#' are comments.
# Each line is a file pattern followed by one or more owners.
# See https://help.github.com/articles/about-codeowners/
# for more info about the CODEOWNERS file
# Team Boto
salt/**/*boto* @saltstack/team-boto
# Team Core
salt/auth/ @saltstack/team-core
salt/cache/ @saltstack/team-core
salt/cli/ @saltstack/team-core
salt/client/* @saltstack/team-core
salt/config/* @saltstack/team-core
salt/daemons/ @saltstack/team-core
salt/pillar/ @saltstack/team-core
salt/loader.py @saltstack/team-core
salt/payload.py @saltstack/team-core
salt/**/master* @saltstack/team-core
salt/**/minion* @saltstack/team-core
# Team Cloud
salt/cloud/ @saltstack/team-cloud
salt/utils/openstack/ @saltstack/team-cloud
salt/utils/aws.py @saltstack/team-cloud
salt/**/*cloud* @saltstack/team-cloud
# Team NetAPI
salt/cli/api.py @saltstack/team-netapi
salt/client/netapi.py @saltstack/team-netapi
salt/netapi/ @saltstack/team-netapi
# Team Network
salt/proxy/ @saltstack/team-proxy
# Team SPM
salt/cli/spm.py @saltstack/team-spm
salt/spm/ @saltstack/team-spm
# Team SSH
salt/cli/ssh.py @saltstack/team-ssh
salt/client/ssh/ @saltstack/team-ssh
salt/runners/ssh.py @saltstack/team-ssh
salt/**/thin.py @saltstack/team-ssh
# Team State
salt/state.py @saltstack/team-state
# Team Transport
salt/transport/ @saltstack/team-transport
salt/utils/zeromq.py @saltstack/team-transport
# Team Windows
salt/**/*win* @saltstack/team-windows

View file

@ -89,7 +89,7 @@ if Defined x (
if %Python%==2 (
Set "PyDir=C:\Python27"
) else (
Set "PyDir=C:\Program Files\Python35"
Set "PyDir=C:\Python35"
)
Set "PATH=%PATH%;%PyDir%;%PyDir%\Scripts"

View file

@ -175,7 +175,7 @@ If (Test-Path "$($ini['Settings']['Python2Dir'])\python.exe") {
DownloadFileWithProgress $url $file
Write-Output " - $script_name :: Installing $($ini[$bitPrograms]['Python2']) . . ."
$p = Start-Process msiexec -ArgumentList "/i $file /qb ADDLOCAL=DefaultFeature,SharedCRT,Extensions,pip_feature,PrependPath TARGETDIR=$($ini['Settings']['Python2Dir'])" -Wait -NoNewWindow -PassThru
$p = Start-Process msiexec -ArgumentList "/i $file /qb ADDLOCAL=DefaultFeature,SharedCRT,Extensions,pip_feature,PrependPath TARGETDIR=`"$($ini['Settings']['Python2Dir'])`"" -Wait -NoNewWindow -PassThru
}
#------------------------------------------------------------------------------
@ -191,7 +191,7 @@ If (!($Path.ToLower().Contains("$($ini['Settings']['Scripts2Dir'])".ToLower())))
#==============================================================================
# Update PIP and SetupTools
# caching depends on environmant variable SALT_PIP_LOCAL_CACHE
# caching depends on environment variable SALT_PIP_LOCAL_CACHE
#==============================================================================
Write-Output " ----------------------------------------------------------------"
Write-Output " - $script_name :: Updating PIP and SetupTools . . ."
@ -212,7 +212,7 @@ if ( ! [bool]$Env:SALT_PIP_LOCAL_CACHE) {
#==============================================================================
# Install pypi resources using pip
# caching depends on environmant variable SALT_REQ_LOCAL_CACHE
# caching depends on environment variable SALT_REQ_LOCAL_CACHE
#==============================================================================
Write-Output " ----------------------------------------------------------------"
Write-Output " - $script_name :: Installing pypi resources using pip . . ."
@ -230,6 +230,24 @@ if ( ! [bool]$Env:SALT_REQ_LOCAL_CACHE) {
Start_Process_and_test_exitcode "$($ini['Settings']['Python2Dir'])\python.exe" "-m pip install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_2.txt" "pip install"
}
#==============================================================================
# Move PyWin32 DLL's to site-packages\win32
#==============================================================================
Write-Output " - $script_name :: Moving PyWin32 DLLs . . ."
Move-Item "$($ini['Settings']['SitePkgs2Dir'])\pywin32_system32\*.dll" "$($ini['Settings']['SitePkgs2Dir'])\win32" -Force
# Remove pywin32_system32 directory
Write-Output " - $script_name :: Removing pywin32_system32 Directory . . ."
Remove-Item "$($ini['Settings']['SitePkgs2Dir'])\pywin32_system32"
# Remove pythonwin directory
Write-Output " - $script_name :: Removing pythonwin Directory . . ."
Remove-Item "$($ini['Settings']['SitePkgs2Dir'])\pythonwin" -Force -Recurse
# Remove PyWin32 PostInstall and testall Scripts
Write-Output " - $script_name :: Removing PyWin32 scripts . . ."
Remove-Item "$($ini['Settings']['Scripts2Dir'])\pywin32_*" -Force -Recurse
#==============================================================================
# Install PyYAML with CLoader
# This has to be a compiled binary to get the CLoader

View file

@ -175,7 +175,7 @@ If (Test-Path "$($ini['Settings']['Python3Dir'])\python.exe") {
DownloadFileWithProgress $url $file
Write-Output " - $script_name :: Installing $($ini[$bitPrograms]['Python3']) . . ."
$p = Start-Process $file -ArgumentList '/passive InstallAllUsers=1 TargetDir="C:\Program Files\Python35" Include_doc=0 Include_tcltk=0 Include_test=0 Include_launcher=0 PrependPath=1 Shortcuts=0' -Wait -NoNewWindow -PassThru
$p = Start-Process $file -ArgumentList "/passive InstallAllUsers=1 TargetDir=`"$($ini['Settings']['Python3Dir'])`" Include_doc=0 Include_tcltk=0 Include_test=0 Include_launcher=0 PrependPath=1 Shortcuts=0" -Wait -NoNewWindow -PassThru
}
#------------------------------------------------------------------------------
@ -247,7 +247,7 @@ Start_Process_and_test_exitcode "$($ini['Settings']['Scripts3Dir'])\pip.exe" "i
# Move DLL's to Python Root
Write-Output " - $script_name :: Moving PyWin32 DLLs . . ."
Move-Item "$($ini['Settings']['SitePkgs3Dir'])\pywin32_system32\*.dll" "$($ini['Settings']['Python3Dir'])" -Force
Move-Item "$($ini['Settings']['SitePkgs3Dir'])\pywin32_system32\*.dll" "$($ini['Settings']['SitePkgs3Dir'])\win32" -Force
# Remove pywin32_system32 directory
Write-Output " - $script_name :: Removing pywin32_system32 Directory . . ."
@ -257,6 +257,10 @@ Remove-Item "$($ini['Settings']['SitePkgs3Dir'])\pywin32_system32"
Write-Output " - $script_name :: Removing pythonwin Directory . . ."
Remove-Item "$($ini['Settings']['SitePkgs3Dir'])\pythonwin" -Force -Recurse
# Remove PyWin32 PostInstall and testall Scripts
Write-Output " - $script_name :: Removing PyWin32 scripts . . ."
Remove-Item "$($ini['Settings']['Scripts3Dir'])\pywin32_*" -Force -Recurse
#==============================================================================
# Fix PyCrypto
#==============================================================================

View file

@ -56,7 +56,7 @@ if %Python%==2 (
Set "PyVerMajor=2"
Set "PyVerMinor=7"
) else (
Set "PyDir=C:\Program Files\Python35"
Set "PyDir=C:\Python35"
Set "PyVerMajor=3"
Set "PyVerMinor=5"
)

View file

@ -16,9 +16,10 @@ if %errorLevel%==0 (
)
echo.
:CheckPython2
if exist "\Python27" goto RemovePython2
if exist "\Program Files\Python35" goto RemovePython3
goto eof
goto CheckPython3
:RemovePython2
rem Uninstall Python 2.7
@ -47,25 +48,30 @@ goto eof
goto eof
:CheckPython3
if exist "\Python35" goto RemovePython3
goto eof
:RemovePython3
echo %0 :: Uninstalling Python 3 ...
echo ---------------------------------------------------------------------
:: 64 bit
if exist "%LOCALAPPDATA%\Package Cache\{b94f45d6-8461-440c-aa4d-bf197b2c2499}" (
echo %0 :: - 3.5.3 64bit
"%LOCALAPPDATA%\Package Cache\{b94f45d6-8461-440c-aa4d-bf197b2c2499}\python-3.5.3-amd64.exe" /uninstall
"%LOCALAPPDATA%\Package Cache\{b94f45d6-8461-440c-aa4d-bf197b2c2499}\python-3.5.3-amd64.exe" /uninstall /passive
)
:: 32 bit
if exist "%LOCALAPPDATA%\Package Cache\{a10037e1-4247-47c9-935b-c5ca049d0299}" (
echo %0 :: - 3.5.3 32bit
"%LOCALAPPDATA%\Package Cache\{a10037e1-4247-47c9-935b-c5ca049d0299}\python-3.5.3" /uninstall
"%LOCALAPPDATA%\Package Cache\{a10037e1-4247-47c9-935b-c5ca049d0299}\python-3.5.3" /uninstall /passive
)
rem wipe the Python directory
echo %0 :: Removing the C:\Program Files\Python35 Directory ...
echo %0 :: Removing the C:\Python35 Directory ...
echo ---------------------------------------------------------------------
rd /s /q "C:\Program Files\Python35"
rd /s /q "C:\Python35"
if %errorLevel%==0 (
echo Successful
) else (

View file

@ -19,9 +19,9 @@ Function Get-Settings {
"Python2Dir" = "C:\Python27"
"Scripts2Dir" = "C:\Python27\Scripts"
"SitePkgs2Dir" = "C:\Python27\Lib\site-packages"
"Python3Dir" = "C:\Program Files\Python35"
"Scripts3Dir" = "C:\Program Files\Python35\Scripts"
"SitePkgs3Dir" = "C:\Program Files\Python35\Lib\site-packages"
"Python3Dir" = "C:\Python35"
"Scripts3Dir" = "C:\Python35\Scripts"
"SitePkgs3Dir" = "C:\Python35\Lib\site-packages"
"DownloadDir" = "$env:Temp\DevSalt"
}
# The script deletes the DownLoadDir (above) for each install.

View file

@ -53,7 +53,7 @@ _DFLT_LOG_DATEFMT = '%H:%M:%S'
_DFLT_LOG_DATEFMT_LOGFILE = '%Y-%m-%d %H:%M:%S'
_DFLT_LOG_FMT_CONSOLE = '[%(levelname)-8s] %(message)s'
_DFLT_LOG_FMT_LOGFILE = (
'%(asctime)s,%(msecs)03d [%(name)-17s][%(levelname)-8s][%(process)d] %(message)s'
'%(asctime)s,%(msecs)03d [%(name)-17s:%(lineno)-4d][%(levelname)-8s][%(process)d] %(message)s'
)
_DFLT_REFSPECS = ['+refs/heads/*:refs/remotes/origin/*', '+refs/tags/*:refs/tags/*']

View file

@ -41,6 +41,7 @@ import salt.utils.immutabletypes as immutabletypes
import salt.utils.url
import salt.utils.platform
import salt.utils.process
import salt.utils.files
import salt.syspaths as syspaths
from salt.template import compile_template, compile_template_str
from salt.exceptions import (
@ -149,6 +150,13 @@ def _gen_tag(low):
return u'{0[state]}_|-{0[__id__]}_|-{0[name]}_|-{0[fun]}'.format(low)
def _clean_tag(tag):
'''
Make tag name safe for filenames
'''
return salt.utils.files.safe_filename_leaf(tag)
def _l_tag(name, id_):
low = {u'name': u'listen_{0}'.format(name),
u'__id__': u'listen_{0}'.format(id_),
@ -1741,7 +1749,7 @@ class State(object):
trb)
}
troot = os.path.join(self.opts[u'cachedir'], self.jid)
tfile = os.path.join(troot, tag)
tfile = os.path.join(troot, _clean_tag(tag))
if not os.path.isdir(troot):
try:
os.makedirs(troot)
@ -2097,7 +2105,7 @@ class State(object):
proc = running[tag].get(u'proc')
if proc:
if not proc.is_alive():
ret_cache = os.path.join(self.opts[u'cachedir'], self.jid, tag)
ret_cache = os.path.join(self.opts[u'cachedir'], self.jid, _clean_tag(tag))
if not os.path.isfile(ret_cache):
ret = {u'result': False,
u'comment': u'Parallel process failed to return',

View file

@ -116,7 +116,7 @@ entry on the minion already contains a numeric value, then using the ``random``
keyword will not modify it.
Added the opportunity to set a job with a special keyword like '@reboot' or
'@hourly'.
'@hourly'. Quotes must be used, otherwise PyYAML will strip the '@' sign.
.. code-block:: yaml
@ -302,7 +302,8 @@ def present(name,
edits. This defaults to the state id
special
A special keyword to specify periodicity (eg. @reboot, @hourly...)
A special keyword to specify periodicity (eg. @reboot, @hourly...).
Quotes must be used, otherwise PyYAML will strip the '@' sign.
.. versionadded:: 2016.3.0
'''
@ -388,7 +389,8 @@ def absent(name,
edits. This defaults to the state id
special
The special keyword used in the job (eg. @reboot, @hourly...)
The special keyword used in the job (eg. @reboot, @hourly...).
Quotes must be used, otherwise PyYAML will strip the '@' sign.
'''
### NOTE: The keyword arguments in **kwargs are ignored in this state, but
### cannot be removed from the function definition, otherwise the use

View file

@ -1565,7 +1565,7 @@ def managed(name,
the salt master and potentially run through a templating system.
name
The location of the file to manage
The location of the file to manage, as an absolute path.
source
The source file to download to the minion, this source file can be
@ -1735,13 +1735,15 @@ def managed(name,
group
The group ownership set for the file, this defaults to the group salt
is running as on the minion On Windows, this is ignored
is running as on the minion. On Windows, this is ignored
mode
The permissions to set on this file, e.g. ``644``, ``0775``, or ``4664``.
The permissions to set on this file, e.g. ``644``, ``0775``, or
``4664``.
The default mode for new files and directories corresponds umask of salt
process. For existing files and directories it's not enforced.
The default mode for new files and directories corresponds to the
umask of the salt process. The mode of existing files and directories
will only be changed if ``mode`` is specified.
.. note::
This option is **not** supported on Windows.
@ -2584,7 +2586,7 @@ def directory(name,
Ensure that a named directory is present and has the right perms
name
The location to create or manage a directory
The location to create or manage a directory, as an absolute path
user
The user to own the directory; this defaults to the user salt is

View file

@ -1435,7 +1435,7 @@ def is_true(value=None):
pass
# Now check for truthiness
if isinstance(value, (int, float)):
if isinstance(value, (six.integer_types, float)):
return value > 0
elif isinstance(value, six.string_types):
return str(value).lower() == 'true'
@ -1914,7 +1914,7 @@ def repack_dictlist(data,
if val_cb is None:
val_cb = lambda x, y: y
valid_non_dict = (six.string_types, int, float)
valid_non_dict = (six.string_types, six.integer_types, float)
if isinstance(data, list):
for element in data:
if isinstance(element, valid_non_dict):

View file

@ -44,7 +44,6 @@ from salt.utils.versions import LooseVersion as _LooseVersion
# Import third party libs
from salt.ext import six
VALID_PROVIDERS = ('pygit2', 'gitpython')
VALID_REF_TYPES = __DEFAULT_MASTER_OPTS['gitfs_ref_types']
# Optional per-remote params that can only be used on a per-remote basis, and
@ -176,7 +175,7 @@ class GitProvider(object):
directly.
self.provider should be set in the sub-class' __init__ function before
invoking GitProvider.__init__().
invoking the parent class' __init__.
'''
def __init__(self, opts, remote, per_remote_defaults, per_remote_only,
override_params, cache_root, role='gitfs'):
@ -931,8 +930,10 @@ class GitPython(GitProvider):
def __init__(self, opts, remote, per_remote_defaults, per_remote_only,
override_params, cache_root, role='gitfs'):
self.provider = 'gitpython'
GitProvider.__init__(self, opts, remote, per_remote_defaults,
per_remote_only, override_params, cache_root, role)
super(GitPython, self).__init__(
opts, remote, per_remote_defaults, per_remote_only,
override_params, cache_root, role
)
def add_refspecs(self, *refspecs):
'''
@ -1266,8 +1267,10 @@ class Pygit2(GitProvider):
def __init__(self, opts, remote, per_remote_defaults, per_remote_only,
override_params, cache_root, role='gitfs'):
self.provider = 'pygit2'
GitProvider.__init__(self, opts, remote, per_remote_defaults,
per_remote_only, override_params, cache_root, role)
super(Pygit2, self).__init__(
opts, remote, per_remote_defaults, per_remote_only,
override_params, cache_root, role
)
def add_refspecs(self, *refspecs):
'''
@ -1952,11 +1955,17 @@ class Pygit2(GitProvider):
fp_.write(blob.data)
GIT_PROVIDERS = {
'pygit2': Pygit2,
'gitpython': GitPython,
}
class GitBase(object):
'''
Base class for gitfs/git_pillar
'''
def __init__(self, opts, valid_providers=VALID_PROVIDERS, cache_root=None):
def __init__(self, opts, git_providers=None, cache_root=None):
'''
IMPORTANT: If specifying a cache_root, understand that this is also
where the remotes will be cloned. A non-default cache_root is only
@ -1964,8 +1973,9 @@ class GitBase(object):
out into the winrepo locations and not within the cachedir.
'''
self.opts = opts
self.valid_providers = valid_providers
self.get_provider()
self.git_providers = git_providers if git_providers is not None \
else GIT_PROVIDERS
self.verify_provider()
if cache_root is not None:
self.cache_root = self.remote_root = cache_root
else:
@ -2023,7 +2033,7 @@ class GitBase(object):
self.remotes = []
for remote in remotes:
repo_obj = self.provider_class(
repo_obj = self.git_providers[self.provider](
self.opts,
remote,
per_remote_defaults,
@ -2277,7 +2287,7 @@ class GitBase(object):
# Hash file won't exist if no files have yet been served up
pass
def get_provider(self):
def verify_provider(self):
'''
Determine which provider to use
'''
@ -2298,12 +2308,12 @@ class GitBase(object):
# Should only happen if someone does something silly like
# set the provider to a numeric value.
desired_provider = str(desired_provider).lower()
if desired_provider not in self.valid_providers:
if desired_provider not in self.git_providers:
log.critical(
'Invalid {0}_provider \'{1}\'. Valid choices are: {2}'
.format(self.role,
desired_provider,
', '.join(self.valid_providers))
', '.join(self.git_providers))
)
failhard(self.role)
elif desired_provider == 'pygit2' and self.verify_pygit2():
@ -2316,17 +2326,13 @@ class GitBase(object):
.format(self.role)
)
failhard(self.role)
if self.provider == 'pygit2':
self.provider_class = Pygit2
elif self.provider == 'gitpython':
self.provider_class = GitPython
def verify_gitpython(self, quiet=False):
'''
Check if GitPython is available and at a compatible version (>= 0.3.0)
'''
def _recommend():
if HAS_PYGIT2 and 'pygit2' in self.valid_providers:
if HAS_PYGIT2 and 'pygit2' in self.git_providers:
log.error(_RECOMMEND_PYGIT2.format(self.role))
if not HAS_GITPYTHON:
@ -2337,7 +2343,7 @@ class GitBase(object):
)
_recommend()
return False
elif 'gitpython' not in self.valid_providers:
elif 'gitpython' not in self.git_providers:
return False
# pylint: disable=no-member
@ -2377,7 +2383,7 @@ class GitBase(object):
Pygit2 must be at least 0.20.3 and libgit2 must be at least 0.20.0.
'''
def _recommend():
if HAS_GITPYTHON and 'gitpython' in self.valid_providers:
if HAS_GITPYTHON and 'gitpython' in self.git_providers:
log.error(_RECOMMEND_GITPYTHON.format(self.role))
if not HAS_PYGIT2:
@ -2388,7 +2394,7 @@ class GitBase(object):
)
_recommend()
return False
elif 'pygit2' not in self.valid_providers:
elif 'pygit2' not in self.git_providers:
return False
# pylint: disable=no-member
@ -2507,7 +2513,7 @@ class GitFS(GitBase):
'''
def __init__(self, opts):
self.role = 'gitfs'
GitBase.__init__(self, opts)
super(GitFS, self).__init__(opts)
def dir_list(self, load):
'''
@ -2791,7 +2797,7 @@ class GitPillar(GitBase):
'''
def __init__(self, opts):
self.role = 'git_pillar'
GitBase.__init__(self, opts)
super(GitPillar, self).__init__(opts)
def checkout(self):
'''
@ -2881,7 +2887,7 @@ class WinRepo(GitBase):
'''
def __init__(self, opts, winrepo_dir):
self.role = 'winrepo'
GitBase.__init__(self, opts, cache_root=winrepo_dir)
super(WinRepo, self).__init__(opts, cache_root=winrepo_dir)
def checkout(self):
'''

View file

@ -36,7 +36,8 @@ class ChefTestCase(TestCase, LoaderModuleMockMixin):
'''
Test if it execute a chef client run and return a dict
'''
self.assertDictEqual(chef.client(), {})
with patch.dict(chef.__opts__, {'cachedir': r'c:\salt\var\cache\salt\minion'}):
self.assertDictEqual(chef.client(), {})
# 'solo' function tests: 1
@ -44,4 +45,5 @@ class ChefTestCase(TestCase, LoaderModuleMockMixin):
'''
Test if it execute a chef solo run and return a dict
'''
self.assertDictEqual(chef.solo('/dev/sda1'), {})
with patch.dict(chef.__opts__, {'cachedir': r'c:\salt\var\cache\salt\minion'}):
self.assertDictEqual(chef.solo('/dev/sda1'), {})

View file

@ -66,7 +66,7 @@ class TestGitFSProvider(TestCase):
('git_pillar', salt.utils.gitfs.GitPillar),
('winrepo', salt.utils.gitfs.WinRepo)):
key = '{0}_provider'.format(role_name)
for provider in salt.utils.gitfs.VALID_PROVIDERS:
for provider in salt.utils.gitfs.GIT_PROVIDERS:
verify = 'verify_gitpython'
mock1 = _get_mock(verify, provider)
with patch.object(role_class, verify, mock1):