Merge pull request #49305 from terminalmage/issue48299

Allow git.latest to remove local tags which have been removed remotely
This commit is contained in:
Daniel Wallace 2018-08-24 12:29:02 -05:00 committed by GitHub
commit b65890c363
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 341 additions and 77 deletions

View file

@ -4845,6 +4845,116 @@ def symbolic_ref(cwd,
output_encoding=output_encoding)['stdout']
def tag(cwd,
name,
ref='HEAD',
message=None,
opts='',
git_opts='',
user=None,
password=None,
ignore_retcode=False,
output_encoding=None):
'''
.. versionadded:: 2018.3.4
Interface to `git-tag(1)`_, adds and removes tags.
cwd
The path to the main git checkout or a linked worktree
name
Name of the tag
ref : HEAD
Which ref to tag (defaults to local clone's HEAD)
.. note::
This argument is ignored when either ``-d`` or ``--delete`` is
present in the ``opts`` passed to this function.
message
Optional message to include with the tag. If provided, an annotated tag
will be created.
opts
Any additional options to add to the command line, in a single string
.. note::
Additionally, on the Salt CLI, if the opts are preceded with a
dash, it is necessary to precede them with ``opts=`` (as in the CLI
examples below) to avoid causing errors with Salt's own argument
parsing.
git_opts
Any additional options to add to git command itself (not the
``worktree`` subcommand), in a single string. This is useful for
passing ``-c`` to run git with temporary changes to the git
configuration.
.. note::
This is only supported in git 1.7.2 and newer.
user
User under which to run the git command. By default, the command is run
by the user under which the minion is running.
password
Windows only. Required when specifying ``user``. This parameter will be
ignored on non-Windows platforms.
ignore_retcode : False
If ``True``, do not log an error to the minion log if the git command
returns a nonzero exit status.
output_encoding
Use this option to specify which encoding to use to decode the output
from any git commands which are run. This should not be needed in most
cases.
.. note::
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. _`git-tag(1)`: http://git-scm.com/docs/git-tag
CLI Example:
.. code-block:: bash
# Create an non-annotated tag
salt myminion git.tag /path/to/repo v1.2
# Create an annotated tag
salt myminion git.tag /path/to/repo v1.2 message='Version 1.2'
# Delete the tag
salt myminion git.tag /path/to/repo v1.2 opts='-d'
'''
cwd = _expand_path(cwd, user)
command = ['git'] + _format_git_opts(git_opts)
command.append('tag')
# Don't add options for annotated commits, since we'll automatically add
# them if a message was passed. This keeps us from blocking on input, since
# passing an annotated command
formatted_opts = [x for x in _format_opts(opts) if x not in ('-a', '--annotate')]
# Make sure that the message was not passed in the opts
if any(x == '-m' or '--message' in x for x in formatted_opts):
raise SaltInvocationError(
'Tag messages must be passed in the "message" argument'
)
command.extend(formatted_opts)
command.append(name)
if '-d' not in formatted_opts and '--delete' not in formatted_opts:
command.append(ref)
return _git_run(command,
cwd=cwd,
user=user,
password=password,
ignore_retcode=ignore_retcode,
redirect_stderr=True,
output_encoding=output_encoding)['stdout']
def version(versioninfo=False):
'''
.. versionadded:: 2015.8.0

View file

@ -158,7 +158,12 @@ def _uptodate(ret, target, comments=None, local_changes=False):
# Shouldn't be making any changes if the repo was up to date, but
# report on them so we are alerted to potential problems with our
# logic.
ret['comment'] += '\n\nChanges made: ' + comments
ret['comment'] += (
'\n\nChanges {0}made: {1}'.format(
'that would be ' if __opts__['test'] else '',
_format_comments(comments)
)
)
return ret
@ -171,8 +176,7 @@ def _neutral_test(ret, comment):
def _fail(ret, msg, comments=None):
ret['result'] = False
if comments:
msg += '\n\nChanges already made: '
msg += _format_comments(comments)
msg += '\n\nChanges already made: ' + _format_comments(comments)
ret['comment'] = msg
return ret
@ -184,8 +188,12 @@ def _already_cloned(ret, target, branch=None, comments=None):
' and is checked out to branch \'{0}\''.format(branch) if branch else ''
)
if comments:
ret['comment'] += '\n\nChanges already made: '
ret['comment'] += _format_comments(comments)
ret['comment'] += (
'\n\nChanges {0}made: {1}'.format(
'that would be ' if __opts__['test'] else '',
_format_comments(comments)
)
)
return ret
@ -268,6 +276,7 @@ def latest(name,
mirror=False,
remote='origin',
fetch_tags=True,
sync_tags=True,
depth=None,
identity=None,
https_user=None,
@ -460,6 +469,12 @@ def latest(name,
If ``True``, then when a fetch is performed all tags will be fetched,
even those which are not reachable by any branch on the remote.
sync_tags : True
If ``True``, then Salt will delete tags which exist in the local clone
but are not found on the remote repository.
.. versionadded:: 2018.3.4
depth
Defines depth in history when git a clone is needed in order to ensure
latest. E.g. ``depth: 1`` is useful when deploying from a repository
@ -851,11 +866,14 @@ def latest(name,
user=user,
password=password,
output_encoding=output_encoding)
all_local_tags = __salt__['git.list_tags'](
target,
user=user,
password=password,
output_encoding=output_encoding)
all_local_tags = set(
__salt__['git.list_tags'](
target,
user=user,
password=password,
output_encoding=output_encoding
)
)
local_rev, local_branch = _get_local_rev_and_branch(
target,
user,
@ -1370,11 +1388,43 @@ def latest(name,
ignore_retcode=True,
output_encoding=output_encoding) if '^{}' not in x
])
if set(all_local_tags) != remote_tags:
if all_local_tags != remote_tags:
has_remote_rev = False
ret['changes']['new_tags'] = list(remote_tags.symmetric_difference(
all_local_tags
))
new_tags = remote_tags - all_local_tags
deleted_tags = all_local_tags - remote_tags
if new_tags:
ret['changes']['new_tags'] = new_tags
if sync_tags and deleted_tags:
# Delete the local copy of the tags to keep up with the
# remote repository.
for tag_name in deleted_tags:
try:
if not __opts__['test']:
__salt__['git.tag'](
target,
tag_name,
opts='-d',
user=user,
password=password,
output_encoding=output_encoding)
except CommandExecutionError as exc:
ret.setdefault('warnings', []).append(
'Failed to remove local tag \'{0}\':\n\n'
'{1}\n\n'.format(tag_name, exc)
)
else:
ret['changes'].setdefault(
'deleted_tags', []).append(tag_name)
if ret['changes'].get('deleted_tags'):
comments.append(
'The following tags {0} removed from the local '
'checkout: {1}'.format(
'would be' if __opts__['test']
else 'were',
', '.join(ret['changes']['deleted_tags'])
)
)
if not has_remote_rev:
try:

View file

@ -8,18 +8,24 @@ from __future__ import absolute_import, print_function, unicode_literals
import functools
import inspect
import os
import shutil
import socket
import string
import tempfile
# Import Salt Testing libs
from tests.support.case import ModuleCase
from tests.support.helpers import with_tempdir
from tests.support.mixins import SaltReturnAssertsMixin
from tests.support.paths import TMP
# Import salt libs
import salt.utils.files
import salt.utils.path
from salt.utils.versions import LooseVersion as _LooseVersion
from salt.ext.six.moves.urllib.parse import urlparse # pylint: disable=no-name-in-module
TEST_REPO = 'https://github.com/saltstack/salt-test-repo.git'
def __check_git_version(caller, min_version, skip_msg):
@ -66,6 +72,9 @@ def ensure_min_git(caller):
def uses_git_opts(caller):
'''
Skip test if git_opts is not supported
IMPORTANT! This decorator should be at the bottom of any decorators added
to a given function.
'''
min_version = '1.7.2'
return __check_git_version(
@ -75,14 +84,63 @@ def uses_git_opts(caller):
)
class WithGitMirror(object):
def __init__(self, repo_url, **kwargs):
self.repo_url = repo_url
if 'dir' not in kwargs:
kwargs['dir'] = TMP
self.kwargs = kwargs
def __call__(self, func):
self.func = func
return functools.wraps(func)(
lambda testcase, *args, **kwargs: self.wrap(testcase, *args, **kwargs) # pylint: disable=W0108
)
def wrap(self, testcase, *args, **kwargs):
# Get temp dir paths
mirror_dir = tempfile.mkdtemp(**self.kwargs)
admin_dir = tempfile.mkdtemp(**self.kwargs)
clone_dir = tempfile.mkdtemp(**self.kwargs)
# Clean up the directories, we want git to actually create them
os.rmdir(mirror_dir)
os.rmdir(admin_dir)
os.rmdir(clone_dir)
# Create a URL to clone
mirror_url = 'file://' + mirror_dir
# Mirror the repo
testcase.run_function(
'git.clone', [mirror_dir], url=TEST_REPO, opts='--mirror')
# Make sure the directory for the mirror now exists
assert os.path.exists(mirror_dir)
# Clone to the admin dir
ret = testcase.run_state('git.latest', name=mirror_url, target=admin_dir)
ret = ret[next(iter(ret))]
assert os.path.exists(admin_dir)
try:
# Run the actual function with three arguments added:
# 1. URL for the test to use to clone
# 2. Cloned admin dir for making/pushing changes to the mirror
# 3. Yet-nonexistant clone_dir for the test function to use as a
# destination for cloning.
return self.func(testcase, mirror_url, admin_dir, clone_dir, *args, **kwargs)
finally:
shutil.rmtree(mirror_dir, ignore_errors=True)
shutil.rmtree(admin_dir, ignore_errors=True)
shutil.rmtree(clone_dir, ignore_errors=True)
with_git_mirror = WithGitMirror
@ensure_min_git
class GitTest(ModuleCase, SaltReturnAssertsMixin):
'''
Validate the git state
'''
def setUp(self):
domain = 'github.com'
self.test_repo = 'https://{0}/saltstack/salt-test-repo.git'.format(domain)
domain = urlparse(TEST_REPO).netloc
try:
if hasattr(socket, 'setdefaulttimeout'):
# 10 second dns timeout
@ -96,6 +154,9 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Reset the dns timeout after the test is over
socket.setdefaulttimeout(None)
def _head(self, cwd):
return self.run_function('git.rev_parse', [cwd, 'HEAD'])
@with_tempdir(create=False)
def test_latest(self, target):
'''
@ -103,7 +164,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
'''
ret = self.run_state(
'git.latest',
name=self.test_repo,
name=TEST_REPO,
target=target
)
self.assertSaltTrueReturn(ret)
@ -116,7 +177,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
'''
ret = self.run_state(
'git.latest',
name=self.test_repo,
name=TEST_REPO,
rev='develop',
target=target,
submodules=True
@ -146,7 +207,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
'''
ret = self.run_state(
'git.latest',
name=self.test_repo,
name=TEST_REPO,
rev='develop',
target=target,
submodules=True
@ -162,7 +223,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
'''
ret = self.run_state(
'git.latest',
name=self.test_repo,
name=TEST_REPO,
rev='develop',
target=target,
unless='test -e {0}'.format(target),
@ -178,7 +239,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
'''
ret = self.run_state(
'git.latest',
name=self.test_repo,
name=TEST_REPO,
rev=0.11,
target=target,
submodules=True,
@ -196,7 +257,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Clone repo
ret = self.run_state(
'git.latest',
name=self.test_repo,
name=TEST_REPO,
target=target
)
self.assertSaltTrueReturn(ret)
@ -212,7 +273,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Re-run state with force_reset=False
ret = self.run_state(
'git.latest',
name=self.test_repo,
name=TEST_REPO,
target=target,
force_reset=False
)
@ -226,7 +287,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Now run the state with force_reset=True
ret = self.run_state(
'git.latest',
name=self.test_repo,
name=TEST_REPO,
target=target,
force_reset=True
)
@ -235,37 +296,21 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Make sure that we no longer have uncommitted changes
self.assertFalse(self.run_function('git.diff', [target, 'HEAD']))
@with_git_mirror(TEST_REPO)
@uses_git_opts
@with_tempdir(create=False)
@with_tempdir(create=False)
@with_tempdir(create=False)
def test_latest_fast_forward(self, mirror_dir, admin_dir, clone_dir):
def test_latest_fast_forward(self, mirror_url, admin_dir, clone_dir):
'''
Test running git.latest state a second time after changes have been
made to the remote repo.
'''
def _head(cwd):
return self.run_function('git.rev_parse', [cwd, 'HEAD'])
mirror_url = 'file://' + mirror_dir
# Mirror the repo
self.run_function(
'git.clone', [mirror_dir], url=self.test_repo, opts='--mirror')
# Make sure the directory for the mirror now exists
self.assertTrue(os.path.exists(mirror_dir))
# Clone the mirror twice, once to the admin location and once to
# the clone_dir
ret = self.run_state('git.latest', name=mirror_url, target=admin_dir)
self.assertSaltTrueReturn(ret)
# Clone the repo
ret = self.run_state('git.latest', name=mirror_url, target=clone_dir)
self.assertSaltTrueReturn(ret)
ret = ret[next(iter(ret))]
assert ret['result']
# Make a change to the repo by editing the file in the admin copy
# of the repo and committing.
head_pre = _head(admin_dir)
head_pre = self._head(admin_dir)
with salt.utils.files.fopen(os.path.join(admin_dir, 'LICENSE'), 'a') as fp_:
fp_.write('Hello world!')
self.run_function(
@ -275,8 +320,8 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
)
# Make sure HEAD is pointing to a new SHA so we know we properly
# committed our change.
head_post = _head(admin_dir)
self.assertNotEqual(head_pre, head_post)
head_post = self._head(admin_dir)
assert head_pre != head_post
# Push the change to the mirror
# NOTE: the test will fail if the salt-test-repo's default branch
@ -285,10 +330,11 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Re-run the git.latest state on the clone_dir
ret = self.run_state('git.latest', name=mirror_url, target=clone_dir)
self.assertSaltTrueReturn(ret)
ret = ret[next(iter(ret))]
assert ret['result']
# Make sure that the clone_dir now has the correct SHA
self.assertEqual(head_post, _head(clone_dir))
assert head_post == self._head(clone_dir)
@with_tempdir(create=False)
def _changed_local_branch_helper(self, target, rev, hint):
@ -299,7 +345,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Clone repo
ret = self.run_state(
'git.latest',
name=self.test_repo,
name=TEST_REPO,
rev=rev,
target=target
)
@ -320,7 +366,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# comment field.
ret = self.run_state(
'git.latest',
name=self.test_repo,
name=TEST_REPO,
rev=rev,
target=target
)
@ -409,7 +455,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
'''
ret = self.run_state(
'git.latest',
name=self.test_repo,
name=TEST_REPO,
rev='HEAD',
target=target,
depth=1
@ -423,7 +469,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
ret = self.run_state(
'git.latest',
name=self.test_repo,
name=TEST_REPO,
rev='non-default-branch',
target=target,
depth=1
@ -431,6 +477,64 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
self.assertSaltTrueReturn(ret)
self.assertTrue(os.path.isdir(os.path.join(target, '.git')))
@with_git_mirror(TEST_REPO)
@uses_git_opts
def test_latest_sync_tags(self, mirror_url, admin_dir, clone_dir):
'''
Test that a removed tag is properly reported as such and removed in the
local clone, and that new tags are reported as new.
'''
tag1 = 'mytag1'
tag2 = 'mytag2'
# Add and push a tag
self.run_function('git.tag', [admin_dir, tag1])
self.run_function('git.push', [admin_dir, 'origin', tag1])
# Clone the repo
ret = self.run_state('git.latest', name=mirror_url, target=clone_dir)
ret = ret[next(iter(ret))]
assert ret['result']
# Now remove the tag
self.run_function('git.push', [admin_dir, 'origin', ':{0}'.format(tag1)])
# Add and push another tag
self.run_function('git.tag', [admin_dir, tag2])
self.run_function('git.push', [admin_dir, 'origin', tag2])
# Re-run the state with sync_tags=False. This should NOT delete the tag
# from the local clone, but should report that a tag has been added.
ret = self.run_state('git.latest',
name=mirror_url,
target=clone_dir,
sync_tags=False)
ret = ret[next(iter(ret))]
assert ret['result']
# Make ABSOLUTELY SURE both tags are present, since we shouldn't have
# removed tag1.
all_tags = self.run_function('git.list_tags', [clone_dir])
assert tag1 in all_tags
assert tag2 in all_tags
# Make sure the reported changes are correct
expected_changes = {'new_tags': [tag2]}
assert ret['changes'] == expected_changes, ret['changes']
# Re-run the state with sync_tags=True. This should remove the local
# tag, since it doesn't exist in the remote repository.
ret = self.run_state('git.latest',
name=mirror_url,
target=clone_dir,
sync_tags=True)
ret = ret[next(iter(ret))]
assert ret['result']
# Make ABSOLUTELY SURE the expected tags are present/gone
all_tags = self.run_function('git.list_tags', [clone_dir])
assert tag1 not in all_tags
assert tag2 in all_tags
# Make sure the reported changes are correct
expected_changes = {'deleted_tags': [tag1]}
assert ret['changes'] == expected_changes, ret['changes']
@with_tempdir(create=False)
def test_cloned(self, target):
'''
@ -439,35 +543,35 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Test mode
ret = self.run_state(
'git.cloned',
name=self.test_repo,
name=TEST_REPO,
target=target,
test=True)
ret = ret[next(iter(ret))]
assert ret['result'] is None
assert ret['changes'] == {
'new': '{0} => {1}'.format(self.test_repo, target)
'new': '{0} => {1}'.format(TEST_REPO, target)
}
assert ret['comment'] == '{0} would be cloned to {1}'.format(
self.test_repo,
TEST_REPO,
target
)
# Now actually run the state
ret = self.run_state(
'git.cloned',
name=self.test_repo,
name=TEST_REPO,
target=target)
ret = ret[next(iter(ret))]
assert ret['result'] is True
assert ret['changes'] == {
'new': '{0} => {1}'.format(self.test_repo, target)
'new': '{0} => {1}'.format(TEST_REPO, target)
}
assert ret['comment'] == '{0} cloned to {1}'.format(self.test_repo, target)
assert ret['comment'] == '{0} cloned to {1}'.format(TEST_REPO, target)
# Run the state again to test idempotence
ret = self.run_state(
'git.cloned',
name=self.test_repo,
name=TEST_REPO,
target=target)
ret = ret[next(iter(ret))]
assert ret['result'] is True
@ -477,7 +581,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Run the state again to test idempotence (test mode)
ret = self.run_state(
'git.cloned',
name=self.test_repo,
name=TEST_REPO,
target=target,
test=True)
ret = ret[next(iter(ret))]
@ -497,18 +601,18 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Test mode
ret = self.run_state(
'git.cloned',
name=self.test_repo,
name=TEST_REPO,
target=target,
branch=old_branch,
test=True)
ret = ret[next(iter(ret))]
assert ret['result'] is None
assert ret['changes'] == {
'new': '{0} => {1}'.format(self.test_repo, target)
'new': '{0} => {1}'.format(TEST_REPO, target)
}
assert ret['comment'] == (
'{0} would be cloned to {1} with branch \'{2}\''.format(
self.test_repo,
TEST_REPO,
target,
old_branch
)
@ -517,17 +621,17 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Now actually run the state
ret = self.run_state(
'git.cloned',
name=self.test_repo,
name=TEST_REPO,
target=target,
branch=old_branch)
ret = ret[next(iter(ret))]
assert ret['result'] is True
assert ret['changes'] == {
'new': '{0} => {1}'.format(self.test_repo, target)
'new': '{0} => {1}'.format(TEST_REPO, target)
}
assert ret['comment'] == (
'{0} cloned to {1} with branch \'{2}\''.format(
self.test_repo,
TEST_REPO,
target,
old_branch
)
@ -536,7 +640,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Run the state again to test idempotence
ret = self.run_state(
'git.cloned',
name=self.test_repo,
name=TEST_REPO,
target=target,
branch=old_branch)
ret = ret[next(iter(ret))]
@ -550,7 +654,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Run the state again to test idempotence (test mode)
ret = self.run_state(
'git.cloned',
name=self.test_repo,
name=TEST_REPO,
target=target,
test=True,
branch=old_branch)
@ -565,7 +669,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Change branch (test mode)
ret = self.run_state(
'git.cloned',
name=self.test_repo,
name=TEST_REPO,
target=target,
branch=new_branch,
test=True)
@ -581,7 +685,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Now really change the branch
ret = self.run_state(
'git.cloned',
name=self.test_repo,
name=TEST_REPO,
target=target,
branch=new_branch)
ret = ret[next(iter(ret))]
@ -598,7 +702,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# locally, as that would fail.
ret = self.run_state(
'git.cloned',
name=self.test_repo,
name=TEST_REPO,
target=target,
branch=old_branch)
ret = ret[next(iter(ret))]
@ -613,7 +717,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Test switching to a nonexistant branch. This should fail.
ret = self.run_state(
'git.cloned',
name=self.test_repo,
name=TEST_REPO,
target=target,
branch=bad_branch)
ret = ret[next(iter(ret))]
@ -633,7 +737,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Test mode
ret = self.run_state(
'git.cloned',
name=self.test_repo,
name=TEST_REPO,
target=target,
branch=branch,
test=True)
@ -642,7 +746,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
assert ret['changes']
assert ret['comment'] == (
'{0} would be cloned to {1} with branch \'{2}\''.format(
self.test_repo,
TEST_REPO,
target,
branch
)
@ -651,7 +755,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
# Now actually run the state
ret = self.run_state(
'git.cloned',
name=self.test_repo,
name=TEST_REPO,
target=target,
branch=branch)
ret = ret[next(iter(ret))]