mirror of
https://github.com/saltstack/salt.git
synced 2025-04-17 10:10:20 +00:00
Merge pull request #29277 from terminalmage/issue29071
Update git_pillar runner to support new git ext_pillar config schema
This commit is contained in:
Mike Place
commit
459d30f27f
2 changed files with 157 additions and 34 deletions
|
|
@ -1,28 +1,105 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
Directly manage the salt git_pillar plugin
|
||||
Runner module to directly manage the git external pillar
|
||||
'''
|
||||
from __future__ import absolute_import
|
||||
|
||||
# Import python libs
|
||||
import logging
|
||||
|
||||
# Import salt libs
|
||||
import salt.pillar.git_pillar
|
||||
import salt.utils.gitfs
|
||||
from salt.exceptions import SaltRunnerError
|
||||
from salt.ext import six
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def update(branch, repo):
|
||||
def update(branch=None, repo=None):
|
||||
'''
|
||||
Execute an update for the configured git fileserver backend for Pillar
|
||||
.. versionadded:: 2014.1.0
|
||||
|
||||
.. versionchanged:: 2015.8.4
|
||||
This runner function now supports the :ref:`new git_pillar
|
||||
configuration schema <git-pillar-2015-8-0-and-later>` introduced in
|
||||
2015.8.0. Additionally, the branch and repo can now be omitted to
|
||||
update all git_pillar remotes. The return data has also changed. For
|
||||
releases 2015.8.3 and earlier, there is no value returned. Starting
|
||||
with 2015.8.4, the return data is a dictionary. If using the :ref:`old
|
||||
git_pillar configuration schema <git-pillar-pre-2015-8-0>`, then the
|
||||
dictionary values will be ``True`` if the update completed without
|
||||
error, and ``False`` if an error occurred. If using the :ref:`new
|
||||
git_pillar configuration schema <git-pillar-2015-8-0-and-later>`, the
|
||||
values will be ``True`` only if new commits were fetched, and ``False``
|
||||
if there were errors or no new commits were fetched.
|
||||
|
||||
Update one or all configured git_pillar remotes.
|
||||
|
||||
CLI Example:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
salt-run git_pillar.update branch='branch' repo='location'
|
||||
# Update specific branch and repo
|
||||
salt-run git_pillar.update branch='branch' repo='https://foo.com/bar.git'
|
||||
# Update all repos (2015.8.4 and later)
|
||||
salt-run git_pillar.update
|
||||
# Run with debug logging
|
||||
salt-run git_pillar.update -l debug
|
||||
'''
|
||||
for opts_dict in __opts__.get('ext_pillar', []):
|
||||
parts = opts_dict.get('git', '').split()
|
||||
if len(parts) >= 2 and parts[:2] == [branch, repo]:
|
||||
salt.pillar.git_pillar.GitPillar(branch, repo, __opts__).update()
|
||||
break
|
||||
else:
|
||||
raise SaltRunnerError('git repo/branch not found in ext_pillar config')
|
||||
ret = {}
|
||||
for ext_pillar in __opts__.get('ext_pillar', []):
|
||||
pillar_type = next(iter(ext_pillar))
|
||||
if pillar_type != 'git':
|
||||
continue
|
||||
pillar_conf = ext_pillar[pillar_type]
|
||||
if isinstance(pillar_conf, six.string_types):
|
||||
parts = pillar_conf.split()
|
||||
if len(parts) >= 2:
|
||||
desired_branch, desired_repo = parts[:2]
|
||||
# Skip this remote if it doesn't match the search criteria
|
||||
if branch is not None:
|
||||
if branch != desired_branch:
|
||||
continue
|
||||
if repo is not None:
|
||||
if repo != desired_repo:
|
||||
continue
|
||||
ret[pillar_conf] = salt.pillar.git_pillar._LegacyGitPillar(
|
||||
parts[0],
|
||||
parts[1],
|
||||
__opts__).update()
|
||||
|
||||
else:
|
||||
pillar = salt.utils.gitfs.GitPillar(__opts__)
|
||||
pillar.init_remotes(pillar_conf,
|
||||
salt.pillar.git_pillar.PER_REMOTE_OVERRIDES)
|
||||
for remote in pillar.remotes:
|
||||
# Skip this remote if it doesn't match the search criteria
|
||||
if branch is not None:
|
||||
if branch != remote.branch:
|
||||
continue
|
||||
if repo is not None:
|
||||
if repo != remote.url:
|
||||
continue
|
||||
try:
|
||||
result = remote.fetch()
|
||||
except Exception as exc:
|
||||
log.error(
|
||||
'Exception \'{0}\' caught while fetching git_pillar '
|
||||
'remote \'{1}\''.format(exc, remote.id),
|
||||
exc_info_on_loglevel=logging.DEBUG
|
||||
)
|
||||
result = False
|
||||
finally:
|
||||
remote.clear_lock()
|
||||
ret[remote.id] = result
|
||||
|
||||
if not ret:
|
||||
if branch is not None or repo is not None:
|
||||
raise SaltRunnerError(
|
||||
'Specified git branch/repo not found in ext_pillar config'
|
||||
)
|
||||
else:
|
||||
raise SaltRunnerError('No git_pillar remotes are configured')
|
||||
|
||||
return ret
|
||||
|
|
|
|||
|
|
@ -298,6 +298,30 @@ class GitProvider(object):
|
|||
_check_ref(ret, base_ref, rname)
|
||||
return ret
|
||||
|
||||
def check_lock(self):
|
||||
'''
|
||||
Used by the provider-specific fetch() function to check the existence
|
||||
of an update lock, and set the lock if not present. If the lock exists
|
||||
already, or if there was a problem setting the lock, this function
|
||||
returns False. If the lock was successfully set, return True.
|
||||
'''
|
||||
if os.path.exists(self.lockfile):
|
||||
log.warning(
|
||||
'Update lockfile is present for {0} remote \'{1}\', '
|
||||
'skipping. If this warning persists, it is possible that the '
|
||||
'update process was interrupted. Removing {2} or running '
|
||||
'\'salt-run cache.clear_git_lock {0}\' will allow updates to '
|
||||
'continue for this remote.'
|
||||
.format(self.role, self.id, self.lockfile)
|
||||
)
|
||||
return False
|
||||
errors = self.lock()[-1]
|
||||
if errors:
|
||||
log.error('Unable to set update lock for {0} remote \'{1}\', '
|
||||
'skipping.'.format(self.role, self.id))
|
||||
return False
|
||||
return True
|
||||
|
||||
def check_root(self):
|
||||
'''
|
||||
Check if the relative root path exists in the checked-out copy of the
|
||||
|
|
@ -347,7 +371,10 @@ class GitProvider(object):
|
|||
else:
|
||||
_add_error(failed, exc)
|
||||
else:
|
||||
msg = 'Removed lock for {0}'.format(self.url)
|
||||
msg = 'Removed lock for {0} remote \'{1}\''.format(
|
||||
self.role,
|
||||
self.id
|
||||
)
|
||||
log.debug(msg)
|
||||
success.append(msg)
|
||||
return success, failed
|
||||
|
|
@ -365,10 +392,13 @@ class GitProvider(object):
|
|||
except (IOError, OSError) as exc:
|
||||
msg = ('Unable to set update lock for {0} ({1}): {2} '
|
||||
.format(self.url, self.lockfile, exc))
|
||||
log.debug(msg)
|
||||
log.error(msg)
|
||||
failed.append(msg)
|
||||
else:
|
||||
msg = 'Set lock for {0}'.format(self.url)
|
||||
msg = 'Set lock for {0} remote \'{1}\''.format(
|
||||
self.role,
|
||||
self.id
|
||||
)
|
||||
log.debug(msg)
|
||||
success.append(msg)
|
||||
return success, failed
|
||||
|
|
@ -579,13 +609,44 @@ class GitPython(GitProvider):
|
|||
Fetch the repo. If the local copy was updated, return True. If the
|
||||
local copy was already up-to-date, return False.
|
||||
'''
|
||||
if not self.check_lock():
|
||||
return False
|
||||
origin = self.repo.remotes[0]
|
||||
try:
|
||||
fetch_results = origin.fetch()
|
||||
except AssertionError:
|
||||
fetch_results = origin.fetch()
|
||||
|
||||
new_objs = False
|
||||
for fetchinfo in fetch_results:
|
||||
if fetchinfo.old_commit is not None:
|
||||
log.debug(
|
||||
'{0} has updated \'{1}\' for remote \'{2}\' '
|
||||
'from {3} to {4}'.format(
|
||||
self.role,
|
||||
fetchinfo.name,
|
||||
self.id,
|
||||
fetchinfo.old_commit.hexsha[:7],
|
||||
fetchinfo.commit.hexsha[:7]
|
||||
)
|
||||
)
|
||||
new_objs = True
|
||||
elif fetchinfo.flags in (fetchinfo.NEW_TAG,
|
||||
fetchinfo.NEW_HEAD):
|
||||
log.debug(
|
||||
'{0} has fetched new {1} \'{2}\' for remote \'{3}\' '
|
||||
.format(
|
||||
self.role,
|
||||
'tag' if fetchinfo.flags == fetchinfo.NEW_TAG
|
||||
else 'head',
|
||||
fetchinfo.name,
|
||||
self.id
|
||||
)
|
||||
)
|
||||
new_objs = True
|
||||
|
||||
cleaned = self.clean_stale_refs()
|
||||
return bool(fetch_results or cleaned)
|
||||
return bool(new_objs or cleaned)
|
||||
|
||||
def file_list(self, tgt_env):
|
||||
'''
|
||||
|
|
@ -938,6 +999,8 @@ class Pygit2(GitProvider):
|
|||
Fetch the repo. If the local copy was updated, return True. If the
|
||||
local copy was already up-to-date, return False.
|
||||
'''
|
||||
if not self.check_lock():
|
||||
return False
|
||||
origin = self.repo.remotes[0]
|
||||
refs_pre = self.repo.listall_references()
|
||||
fetch_kwargs = {}
|
||||
|
|
@ -1277,6 +1340,8 @@ class Dulwich(GitProvider): # pylint: disable=abstract-method
|
|||
Fetch the repo. If the local copy was updated, return True. If the
|
||||
local copy was already up-to-date, return False.
|
||||
'''
|
||||
if not self.check_lock():
|
||||
return False
|
||||
# origin is just a url here, there is no origin object
|
||||
origin = self.url
|
||||
client, path = \
|
||||
|
|
@ -1753,24 +1818,6 @@ class GitBase(object):
|
|||
'''
|
||||
changed = False
|
||||
for repo in self.remotes:
|
||||
if os.path.exists(repo.lockfile):
|
||||
log.warning(
|
||||
'Update lockfile is present for {0} remote \'{1}\', '
|
||||
'skipping. If this warning persists, it is possible that '
|
||||
'the update process was interrupted. Removing {2} or '
|
||||
'running \'salt-run cache.clear_git_lock {0}\' will '
|
||||
'allow updates to continue for this remote.'
|
||||
.format(self.role, repo.id, repo.lockfile)
|
||||
)
|
||||
continue
|
||||
_, errors = repo.lock()
|
||||
if errors:
|
||||
log.error('Unable to set update lock for {0} remote \'{1}\', '
|
||||
'skipping.'.format(self.role, repo.id))
|
||||
continue
|
||||
log.debug(
|
||||
'{0} is fetching from \'{1}\''.format(self.role, repo.id)
|
||||
)
|
||||
try:
|
||||
if repo.fetch():
|
||||
# We can't just use the return value from repo.fetch()
|
||||
|
|
@ -1780,7 +1827,6 @@ class GitBase(object):
|
|||
# this value and make it incorrect.
|
||||
changed = True
|
||||
except Exception as exc:
|
||||
# Do not use {0} in the error message, as exc is not a string
|
||||
log.error(
|
||||
'Exception \'{0}\' caught while fetching {1} remote '
|
||||
'\'{2}\''.format(exc, self.role, repo.id),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue