saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Added debugging outputs

Browse source
This commit is contained in:
David Murphy 2025-01-30 17:04:05 -07:00 • committed by Daniel Wozniak
parent b0029609ab
commit d0231f33ee
3 changed files with 311 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
salt/runners/git_pillar.py
View file

@ -61,12 +61,20 @@ def update(branch=None, repo=None):
# Run with debug logging # Run with debug logging
salt-run git_pillar.update -l debug salt-run git_pillar.update -l debug
""" """
print(
f"DGM runners git_pillar update entry, branch '{branch}', repo '{repo}'",
flush=True,
)
ret = {} ret = {}
for ext_pillar in __opts__.get("ext_pillar", []): for ext_pillar in __opts__.get("ext_pillar", []):
pillar_type = next(iter(ext_pillar)) pillar_type = next(iter(ext_pillar))
if pillar_type != "git": if pillar_type != "git":
continue continue
pillar_conf = ext_pillar[pillar_type] pillar_conf = ext_pillar[pillar_type]
print(
f"DGM runners git_pillar update, pillar_type '{pillar_type}', pillar_conf '{pillar_conf}'",
flush=True,
)
pillar = salt.utils.gitfs.GitPillar( pillar = salt.utils.gitfs.GitPillar(
__opts__, __opts__,
pillar_conf, pillar_conf,
@ -75,6 +83,12 @@ def update(branch=None, repo=None):
global_only=salt.pillar.git_pillar.GLOBAL_ONLY, global_only=salt.pillar.git_pillar.GLOBAL_ONLY,
) )
for remote in pillar.remotes: for remote in pillar.remotes:
dgm_remote_name = getattr(remote, "name", None)
print(
f"DGM runners git_pillar update, branch '{branch}', remote branch '{remote.branch}', remote url '{remote.url}', remote name '{dgm_remote_name}'",
flush=True,
)
# Skip this remote if it doesn't match the search criteria # Skip this remote if it doesn't match the search criteria
if branch is not None: if branch is not None:
if branch != remote.branch: if branch != remote.branch:
@ -104,4 +118,5 @@ def update(branch=None, repo=None):
else: else:
raise SaltRunnerError("No git_pillar remotes are configured") raise SaltRunnerError("No git_pillar remotes are configured")
print(f"DGM runners git_pillar update, exit ret '{ret}'", flush=True)
return ret return ret

213
salt/utils/gitfs.py
View file

@ -19,6 +19,7 @@ import shutil
import stat import stat
import subprocess import subprocess
import time import time
import traceback # # DGM
import weakref import weakref
from datetime import datetime from datetime import datetime
@ -255,6 +256,13 @@ class GitProvider:
self.opts = opts self.opts = opts
self.role = role self.role = role
## DGM
if hasattr(self, "name"):
print(
f"DGM class GitProvider dunder init, very start, hasattr self.name '{self.name}'",
flush=True,
)
def _val_cb(x, y): def _val_cb(x, y):
return str(y) return str(y)
@ -476,21 +484,48 @@ class GitProvider:
failhard(self.role) failhard(self.role)
if hasattr(self, "name"): if hasattr(self, "name"):
self._cache_basehash = self.name self._cache_basehash = self.name
## DGM
print(
f"DGM class GitProvider dunder init, hasattr self.name '{self.name}'",
flush=True,
)
else: else:
hash_type = getattr(hashlib, self.opts.get("hash_type", DEFAULT_HASH_TYPE)) hash_type = getattr(hashlib, self.opts.get("hash_type", DEFAULT_HASH_TYPE))
# We loaded this data from yaml configuration files, so, its safe # We loaded this data from yaml configuration files, so, its safe
# to use UTF-8 # to use UTF-8
## DGM
dgm_orig_cache_bash_str = str(
base64.b64encode(hash_type(self.id.encode("utf-8")).digest()),
encoding="ascii",
)
print(
f"DGM class GitProvider dunder init, hasattr dgm_orig_cache_bash_str '{dgm_orig_cache_bash_str}'",
flush=True,
)
self._cache_basehash = str( self._cache_basehash = str(
base64.b64encode(hash_type(self.id.encode("utf-8")).digest()), base64.b64encode(hash_type(self.id.encode("utf-8")).digest()),
encoding="ascii", # base64 only outputs ascii encoding="ascii", # base64 only outputs ascii
).replace( ).replace(
"/", "_" "/", "_"
) # replace "/" with "_" to not cause trouble with file system ) # replace "/" with "_" to not cause trouble with file system
## DGM
print(
f"DGM class GitProvider dunder init, self._cache_basehash '{self._cache_basehash}'",
flush=True,
)
self._cache_hash = salt.utils.path.join(cache_root, self._cache_basehash) self._cache_hash = salt.utils.path.join(cache_root, self._cache_basehash)
self._cache_basename = "_" self._cache_basename = "_"
if self.id.startswith("__env__"): if self.id.startswith("__env__"):
try: try:
self._cache_basename = self.get_checkout_target() self._cache_basename = self.get_checkout_target()
print(
f"DGM class GitProvider dunder init, self.id '{self.id}', self._cache_basename '{self._cache_basename}'",
flush=True,
)
except AttributeError: except AttributeError:
log.critical( log.critical(
"__env__ cant generate basename: %s %s", self.role, self.id "__env__ cant generate basename: %s %s", self.role, self.id
@ -771,10 +806,17 @@ class GitProvider:
success = [] success = []
failed = [] failed = []
## DGM someone is calling this after the lock file has been removed, ie twice ?
stk_summary = traceback.format_stack()
try: try:
os.remove(lock_file) os.remove(lock_file)
except OSError as exc: except OSError as exc:
if exc.errno == errno.ENOENT: if exc.errno == errno.ENOENT:
print(
f"DGM class GitProvider _clear_lock entry, exception, stk_summary '{stk_summary}'",
flush=True,
)
# No lock file present # No lock file present
msg = ( msg = (
f"Attempt to remove lock {self.url} for file ({lock_file}) " f"Attempt to remove lock {self.url} for file ({lock_file}) "
@ -793,6 +835,12 @@ class GitProvider:
else: else:
_add_error(failed, exc) _add_error(failed, exc)
else: else:
## DGM
print(
f"DGM class GitProvider _clear_lock entry, else, stk_summary '{stk_summary}'",
flush=True,
)
msg = ( msg = (
f"Removed {lock_type} lock for {self.role} remote '{self.id}' " f"Removed {lock_type} lock for {self.role} remote '{self.id}' "
f"on machine_id '{self.mach_id}'" f"on machine_id '{self.mach_id}'"
@ -826,6 +874,10 @@ class GitProvider:
# section for the remote to the git config # section for the remote to the git config
conf.add_section(remote_section) conf.add_section(remote_section)
conf_changed = True conf_changed = True
print(
f"DGM class GitProvider enforce_git_config, URL NoSectionError conf_changed set True '{conf_changed}'",
flush=True,
)
url = None url = None
log.debug( log.debug(
"Current fetch URL for %s remote '%s': %s (desired: %s)", "Current fetch URL for %s remote '%s': %s (desired: %s)",
@ -843,6 +895,10 @@ class GitProvider:
self.url, self.url,
) )
conf_changed = True conf_changed = True
print(
f"DGM class GitProvider enforce_git_config, URL change conf_changed set True '{conf_changed}'",
flush=True,
)
# 2. refspecs # 2. refspecs
try: try:
@ -869,6 +925,10 @@ class GitProvider:
desired_refspecs, desired_refspecs,
) )
conf_changed = True conf_changed = True
print(
f"DGM class GitProvider enforce_git_config, refspecs conf_changed set True '{conf_changed}'",
flush=True,
)
# 3. http.sslVerify # 3. http.sslVerify
try: try:
@ -895,9 +955,17 @@ class GitProvider:
desired_ssl_verify, desired_ssl_verify,
) )
conf_changed = True conf_changed = True
print(
f"DGM class GitProvider enforce_git_config, ssl_verify conf_changed set True '{conf_changed}'",
flush=True,
)
# Write changes, if necessary # Write changes, if necessary
if conf_changed: if conf_changed:
print(
f"DGM class GitProvider enforce_git_config, conf_changed set True, examine file '{git_config}'",
flush=True,
)
with salt.utils.files.fopen(git_config, "w") as fp_: with salt.utils.files.fopen(git_config, "w") as fp_:
conf.write(fp_) conf.write(fp_)
log.debug( log.debug(
@ -915,6 +983,8 @@ class GitProvider:
This function requires that a _fetch() function be implemented in a This function requires that a _fetch() function be implemented in a
sub-class. sub-class.
""" """
## DGM stk_summary = traceback.format_stack()
## DGM print(f"DGM class GitProvider fetch, stk_summary '{stk_summary}'", flush=True)
try: try:
with self.gen_lock(lock_type="update"): with self.gen_lock(lock_type="update"):
log.debug("Fetching %s remote '%s'", self.role, self.id) log.debug("Fetching %s remote '%s'", self.role, self.id)
@ -1565,13 +1635,36 @@ class GitPython(GitProvider):
local copy was already up-to-date, return False. local copy was already up-to-date, return False.
""" """
origin = self.repo.remotes[0] origin = self.repo.remotes[0]
print(f"DGM class GitPython _fetch entry, origin '{dir(origin)}'", flush=True)
try: try:
fetch_results = origin.fetch() fetch_results = origin.fetch()
except AssertionError: except AssertionError:
fetch_results = origin.fetch() fetch_results = origin.fetch()
new_objs = False new_objs = False
print(
f"DGM class GitPython _fetch, origin '{dir(origin)}', fetch_results '{fetch_results}'",
flush=True,
)
for fetchinfo in fetch_results: for fetchinfo in fetch_results:
## DGM
dgm_commit = 0
dgm_old_commit = 0
## DGM
if fetchinfo.old_commit is not None:
dgm_commit = fetchinfo.commit.hexsha[:7]
dgm_old_commit = fetchinfo.old_commit.hexsha[:7]
print(
f"DGM class GitPython _fetch, fetchinfo dir '{dir(fetchinfo)}', self.id '{self.id}', fetchinfo.name '{fetchinfo.name}', fetchinfo.old_commit '{dgm_old_commit}', fetchinfo.commit '{dgm_commit}', fetchinfo.flags '{fetchinfo.flags}', NEW_TAG '{fetchinfo.NEW_TAG}', NEW_HEAD '{fetchinfo.NEW_HEAD}'",
flush=True,
)
if fetchinfo.old_commit is not None: if fetchinfo.old_commit is not None:
log.debug( log.debug(
"%s has updated '%s' for remote '%s' from %s to %s", "%s has updated '%s' for remote '%s' from %s to %s",
@ -1593,7 +1686,10 @@ class GitPython(GitProvider):
new_objs = True new_objs = True
cleaned = self.clean_stale_refs() cleaned = self.clean_stale_refs()
return True if (new_objs or cleaned) else None ## DGM return True if (new_objs or cleaned) else None
dgm_ret = True if (new_objs or cleaned) else None
print(f"DGM class GitPython _fetch, return '{dgm_ret}'", flush=True)
return dgm_ret
def file_list(self, tgt_env): def file_list(self, tgt_env):
""" """
@ -1766,12 +1862,18 @@ class Pygit2(GitProvider):
fetch_on_fail fetch_on_fail
If checkout fails perform a fetch then try to checkout again. If checkout fails perform a fetch then try to checkout again.
""" """
print("DGM class Pygit2 checkout entry", flush=True)
self.fetch_request_check() self.fetch_request_check()
tgt_ref = self.get_checkout_target() tgt_ref = self.get_checkout_target()
local_ref = "refs/heads/" + tgt_ref local_ref = "refs/heads/" + tgt_ref
remote_ref = "refs/remotes/origin/" + tgt_ref remote_ref = "refs/remotes/origin/" + tgt_ref
tag_ref = "refs/tags/" + tgt_ref tag_ref = "refs/tags/" + tgt_ref
print(
f"DGM class Pygit2 checkout entry, tgt_ref '{tgt_ref}', local_ref '{local_ref}', tag_ref '{tag_ref}'",
flush=True,
)
try: try:
local_head = self.repo.lookup_reference("HEAD") local_head = self.repo.lookup_reference("HEAD")
except KeyError: except KeyError:
@ -1854,6 +1956,12 @@ class Pygit2(GitProvider):
# Only perform a checkout if HEAD and target are not pointing # Only perform a checkout if HEAD and target are not pointing
# at the same SHA1. # at the same SHA1.
print(
f"DGM class Pygit2 checkout, head_sha '{head_sha}', target_sha '{target_sha}'",
flush=True,
)
if head_sha != target_sha: if head_sha != target_sha:
# Check existence of the ref in refs/heads/ which # Check existence of the ref in refs/heads/ which
# corresponds to the local HEAD. Checking out local_ref # corresponds to the local HEAD. Checking out local_ref
@ -2108,6 +2216,12 @@ class Pygit2(GitProvider):
origin = self.repo.remotes[0] origin = self.repo.remotes[0]
refs_pre = self.repo.listall_references() refs_pre = self.repo.listall_references()
fetch_kwargs = {} fetch_kwargs = {}
print(
f"DGM class pygit2 _fetch, origin '{dir(origin)}', refs_pre '{refs_pre}'",
flush=True,
)
# pygit2 radically changed fetchiing in 0.23.2 # pygit2 radically changed fetchiing in 0.23.2
if self.remotecallbacks is not None: if self.remotecallbacks is not None:
fetch_kwargs["callbacks"] = self.remotecallbacks fetch_kwargs["callbacks"] = self.remotecallbacks
@ -2121,8 +2235,18 @@ class Pygit2(GitProvider):
pass pass
try: try:
fetch_results = origin.fetch(**fetch_kwargs) fetch_results = origin.fetch(**fetch_kwargs)
print(
f"DGM class pygit2 _fetch, fetch_results '{fetch_results}', fetch_kwargs '{fetch_kwargs}', dir fetch_results '{dir(fetch_results)}'",
flush=True,
)
except GitError as exc: # pylint: disable=broad-except except GitError as exc: # pylint: disable=broad-except
exc_str = get_error_message(exc).lower() exc_str = get_error_message(exc).lower()
print(
f"DGM class pygit2 _fetch, exception giterror exc '{exc_str}'",
flush=True,
)
if "unsupported url protocol" in exc_str and isinstance( if "unsupported url protocol" in exc_str and isinstance(
self.credentials, pygit2.Keypair self.credentials, pygit2.Keypair
): ):
@ -2151,6 +2275,10 @@ class Pygit2(GitProvider):
exc, exc,
exc_info=True, exc_info=True,
) )
print(
f"DGM class pygit2 _fetch, exception giterror returning False, role '{self.role}', id '{self.id}'",
flush=True,
)
return False return False
try: try:
# pygit2.Remote.fetch() returns a dict in pygit2 < 0.21.0 # pygit2.Remote.fetch() returns a dict in pygit2 < 0.21.0
@ -2159,6 +2287,11 @@ class Pygit2(GitProvider):
# pygit2.Remote.fetch() returns a class instance in # pygit2.Remote.fetch() returns a class instance in
# pygit2 >= 0.21.0 # pygit2 >= 0.21.0
received_objects = fetch_results.received_objects received_objects = fetch_results.received_objects
print(
f"DGM class pygit2 _fetch, fetch_results contents, indexed_deltas '{fetch_results.indexed_deltas}', indexed_objects '{fetch_results.indexed_objects}', local_objects '{fetch_results.local_objects}', received_bytes '{fetch_results.received_bytes}', received_objects '{fetch_results.received_objects}', total_deltas '{fetch_results.total_deltas}', total_objects '{fetch_results.total_objects}'",
flush=True,
)
if received_objects != 0: if received_objects != 0:
log.debug( log.debug(
"%s received %s objects for remote '%s'", "%s received %s objects for remote '%s'",
@ -2170,12 +2303,21 @@ class Pygit2(GitProvider):
log.debug("%s remote '%s' is up-to-date", self.role, self.id) log.debug("%s remote '%s' is up-to-date", self.role, self.id)
refs_post = self.repo.listall_references() refs_post = self.repo.listall_references()
cleaned = self.clean_stale_refs(local_refs=refs_post) cleaned = self.clean_stale_refs(local_refs=refs_post)
return True if (received_objects or refs_pre != refs_post or cleaned) else None
## DGM return True if (received_objects or refs_pre != refs_post or cleaned) else None
dgm_ret = (
True if (received_objects or refs_pre != refs_post or cleaned) else None
)
print(
f"DGM class pygit2 _fetch, return '{dgm_ret}', received_objects '{received_objects}', refs_pre '{refs_pre}', refs_post '{refs_post}', cleaned '{cleaned}'",
flush=True,
)
def file_list(self, tgt_env): def file_list(self, tgt_env):
""" """
Get file list for the target environment using pygit2 Get file list for the target environment using pygit2
""" """
print(f"DGM class Pygit2 file_list entry, tgt_env '{tgt_env}'", flush=True)
def _traverse(tree, blobs, prefix): def _traverse(tree, blobs, prefix):
""" """
@ -2540,6 +2682,10 @@ class GitBase:
gitfs.fetch_remotes() gitfs.fetch_remotes()
""" """
print(
f"DGM class GitBase dunder init entry, opts '{opts}', git_providers '{git_providers}', remotes '{remotes}'",
flush=True,
)
self.opts = opts self.opts = opts
self.git_providers = ( self.git_providers = (
git_providers if git_providers is not None else GIT_PROVIDERS git_providers if git_providers is not None else GIT_PROVIDERS
@ -2579,6 +2725,11 @@ class GitBase:
# are defined and the provider is not one that supports auth, then # are defined and the provider is not one that supports auth, then
# error out and do not proceed. # error out and do not proceed.
override_params = copy.deepcopy(per_remote_overrides) override_params = copy.deepcopy(per_remote_overrides)
print(
f"DGM class GitBase init_remotes entry, remotes '{remotes}', override_params '{override_params}'",
flush=True,
)
global_auth_params = [ global_auth_params = [
f"{self.role}_{x}" for x in AUTH_PARAMS if self.opts[f"{self.role}_{x}"] f"{self.role}_{x}" for x in AUTH_PARAMS if self.opts[f"{self.role}_{x}"]
] ]
@ -2754,12 +2905,20 @@ class GitBase:
# matches or else skip this one # matches or else skip this one
try: try:
if not fnmatch.fnmatch(repo.url, remote): if not fnmatch.fnmatch(repo.url, remote):
print(
f"DGM clear_lock not fnmatch, repo.url '{repo.url}', remote '{remote}'",
flush=True,
)
continue continue
except TypeError: except TypeError:
# remote was non-string, try again # remote was non-string, try again
if not fnmatch.fnmatch(repo.url, str(remote)): if not fnmatch.fnmatch(repo.url, str(remote)):
continue continue
success, failed = repo.clear_lock(lock_type=lock_type) success, failed = repo.clear_lock(lock_type=lock_type)
print(
f"DGM clear_lock, success '{success}', failed '{failed}', for lock_type '{lock_type}'",
flush=True,
)
cleared.extend(success) cleared.extend(success)
errors.extend(failed) errors.extend(failed)
@ -2770,6 +2929,7 @@ class GitBase:
Fetch all remotes and return a boolean to let the calling function know Fetch all remotes and return a boolean to let the calling function know
whether or not any remotes were updated in the process of fetching whether or not any remotes were updated in the process of fetching
""" """
print(f"DGM class GitBase fetch_remotes entry, remotes '{remotes}'", flush=True)
if remotes is None: if remotes is None:
remotes = [] remotes = []
elif isinstance(remotes, str): elif isinstance(remotes, str):
@ -2785,18 +2945,34 @@ class GitBase:
changed = False changed = False
for repo in self.remotes: for repo in self.remotes:
name = getattr(repo, "name", None) name = getattr(repo, "name", None)
print(
f"DGM class GitBase fetch_remotes, self.role '{self.role}', name '{name}', repo id '{repo.id}'",
flush=True,
)
if not remotes or (repo.id, name) in remotes or name in remotes: if not remotes or (repo.id, name) in remotes or name in remotes:
try: try:
# Find and place fetch_request file for all the other branches for this repo # Find and place fetch_request file for all the other branches for this repo
repo_work_hash = os.path.split(repo.get_salt_working_dir())[0] repo_work_hash = os.path.split(repo.get_salt_working_dir())[0]
print(
f"DGM class GitBase fetch_remotes, repo working_dir '{repo.get_salt_working_dir()}', repo_work_hash '{repo_work_hash}'",
flush=True,
)
for branch in os.listdir(repo_work_hash): for branch in os.listdir(repo_work_hash):
# Don't place fetch request in current branch being updated # Don't place fetch request in current branch being updated
print(
f"DGM class GitBase fetch_remotes, for loop branch, branch '{branch}', repo cache basename '{repo.get_cache_basename()}'",
flush=True,
)
if branch == repo.get_cache_basename(): if branch == repo.get_cache_basename():
continue continue
branch_salt_dir = salt.utils.path.join(repo_work_hash, branch) branch_salt_dir = salt.utils.path.join(repo_work_hash, branch)
fetch_path = salt.utils.path.join( fetch_path = salt.utils.path.join(
branch_salt_dir, "fetch_request" branch_salt_dir, "fetch_request"
) )
print(
f"DGM class GitBase fetch_remotes, for loop branch, branch_salt_dir '{branch_salt_dir}', fetch_path '{fetch_path}'",
flush=True,
)
if os.path.isdir(branch_salt_dir): if os.path.isdir(branch_salt_dir):
try: try:
with salt.utils.files.fopen(fetch_path, "w"): with salt.utils.files.fopen(fetch_path, "w"):
@ -2818,6 +2994,10 @@ class GitBase:
# changes would override this value and make it # changes would override this value and make it
# incorrect. # incorrect.
changed = True changed = True
print(
f"DGM class GitBase fetch_remotes, for loop branch, repo.fetch check, changed '{changed}'",
flush=True,
)
except Exception as exc: # pylint: disable=broad-except except Exception as exc: # pylint: disable=broad-except
log.error( log.error(
"Exception caught while fetching %s remote '%s': %s", "Exception caught while fetching %s remote '%s': %s",
@ -2826,6 +3006,10 @@ class GitBase:
exc, exc,
exc_info=True, exc_info=True,
) )
print(
f"DGM class GitBase fetch_remotes, exit returning changed '{changed}'",
flush=True,
)
return changed return changed
def lock(self, remote=None): def lock(self, remote=None):
@ -2840,12 +3024,17 @@ class GitBase:
# matches or else skip this one # matches or else skip this one
try: try:
if not fnmatch.fnmatch(repo.url, remote): if not fnmatch.fnmatch(repo.url, remote):
print(
f"DGM lock not fnmatch, repo.url '{repo.url}', remote '{remote}'",
flush=True,
)
continue continue
except TypeError: except TypeError:
# remote was non-string, try again # remote was non-string, try again
if not fnmatch.fnmatch(repo.url, str(remote)): if not fnmatch.fnmatch(repo.url, str(remote)):
continue continue
success, failed = repo.lock() success, failed = repo.lock()
print(f"DGM lock, success '{success}', failed '{failed}'", flush=True)
locked.extend(success) locked.extend(success)
errors.extend(failed) errors.extend(failed)
return locked, errors return locked, errors
@ -2918,6 +3107,8 @@ class GitBase:
""" """
Determine which provider to use Determine which provider to use
""" """
print(f"DGM class GitBase verify_provider, self.role '{self.role}'", flush=True)
if f"verified_{self.role}_provider" in self.opts: if f"verified_{self.role}_provider" in self.opts:
self.provider = self.opts[f"verified_{self.role}_provider"] self.provider = self.opts[f"verified_{self.role}_provider"]
else: else:
@ -3074,10 +3265,20 @@ class GitBase:
fetch_on_fail fetch_on_fail
If checkout fails perform a fetch then try to checkout again. If checkout fails perform a fetch then try to checkout again.
""" """
print(
f"DGM class GitBase do_checkout entry, repo '{repo}', dir repo '{dir(repo)}', fetch_on_fail '{fetch_on_fail}'",
flush=True,
)
time_start = time.time() time_start = time.time()
while time.time() - time_start <= 5: while time.time() - time_start <= 5:
try: try:
return repo.checkout(fetch_on_fail=fetch_on_fail) ## DGM return repo.checkout(fetch_on_fail=fetch_on_fail)
dgm_cachedir = repo.checkout(fetch_on_fail=fetch_on_fail)
print(
f"DGM class GitBase do_checkout, returning cachedir '{dgm_cachedir}'",
flush=True,
)
return dgm_cachedir
except GitLockError as exc: except GitLockError as exc:
if exc.errno == errno.EEXIST: if exc.errno == errno.EEXIST:
time.sleep(0.1) time.sleep(0.1)
@ -3483,8 +3684,14 @@ class GitPillar(GitBase):
""" """
self.pillar_dirs = OrderedDict() self.pillar_dirs = OrderedDict()
self.pillar_linked_dirs = [] self.pillar_linked_dirs = []
print(f"DGM class GitPillar checkout, remotes '{self.remotes}'", flush=True)
for repo in self.remotes: for repo in self.remotes:
cachedir = self.do_checkout(repo, fetch_on_fail=fetch_on_fail) cachedir = self.do_checkout(repo, fetch_on_fail=fetch_on_fail)
print(
f"DGM class GitPillar checkout, repo '{repo}', cachedir '{cachedir}'",
flush=True,
)
if cachedir is not None: if cachedir is not None:
# Figure out which environment this remote should be assigned # Figure out which environment this remote should be assigned
if repo.branch == "__env__" and hasattr(repo, "all_saltenvs"): if repo.branch == "__env__" and hasattr(repo, "all_saltenvs"):

119
tests/pytests/functional/utils/gitfs/test_pillar.py
View file

@ -32,6 +32,10 @@ except ImportError:
skipif_no_gitpython = pytest.mark.skipif(not HAS_GITPYTHON, reason="Missing gitpython") skipif_no_gitpython = pytest.mark.skipif(not HAS_GITPYTHON, reason="Missing gitpython")
skipif_no_pygit2 = pytest.mark.skipif(not HAS_PYGIT2, reason="Missing pygit2") skipif_no_pygit2 = pytest.mark.skipif(not HAS_PYGIT2, reason="Missing pygit2")
## DGM
## testgitfs = "https://github.com/saltstack/salt-test-pillar-gitfs.git"
testgitfs = "https://github.com/dmurphy18/salt-test-pillar-gitfs.git"
@pytest.fixture @pytest.fixture
def pillar_opts(salt_factories, tmp_path): def pillar_opts(salt_factories, tmp_path):
@ -72,9 +76,10 @@ def _get_pillar(opts, *remotes):
@skipif_no_gitpython @skipif_no_gitpython
def test_gitpython_pillar_provider(gitpython_pillar_opts): def test_gitpython_pillar_provider(gitpython_pillar_opts):
p = _get_pillar( ## DGM p = _get_pillar(
gitpython_pillar_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" ## DGM gitpython_pillar_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git"
) ## DGM )
p = _get_pillar(gitpython_pillar_opts, testgitfs)
assert len(p.remotes) == 1 assert len(p.remotes) == 1
assert p.provider == "gitpython" assert p.provider == "gitpython"
assert isinstance(p.remotes[0], GitPython) assert isinstance(p.remotes[0], GitPython)
@ -82,18 +87,20 @@ def test_gitpython_pillar_provider(gitpython_pillar_opts):
@skipif_no_pygit2 @skipif_no_pygit2
def test_pygit2_pillar_provider(pygit2_pillar_opts): def test_pygit2_pillar_provider(pygit2_pillar_opts):
p = _get_pillar( ## DGM p = _get_pillar(
pygit2_pillar_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" ## DGM pygit2_pillar_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git"
) ## DGM )
p = _get_pillar(pygit2_pillar_opts, testgitfs)
assert len(p.remotes) == 1 assert len(p.remotes) == 1
assert p.provider == "pygit2" assert p.provider == "pygit2"
assert isinstance(p.remotes[0], Pygit2) assert isinstance(p.remotes[0], Pygit2)
def _test_env(opts): def _test_env(opts):
p = _get_pillar( ## DGM p = _get_pillar(
opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git" ## DGM opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git"
) ## DGM )
p = _get_pillar(opts, f"__env__ {testgitfs}")
assert len(p.remotes) == 1 assert len(p.remotes) == 1
p.checkout() p.checkout()
repo = p.remotes[0] repo = p.remotes[0]
@ -102,9 +109,10 @@ def _test_env(opts):
for f in (".gitignore", "README.md", "file.sls", "top.sls"): for f in (".gitignore", "README.md", "file.sls", "top.sls"):
assert f in files assert f in files
opts["pillarenv"] = "main" opts["pillarenv"] = "main"
p2 = _get_pillar( ## DGM p2 = _get_pillar(
opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git" ## DGM opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git"
) ## DGM )
p2 = _get_pillar(opts, f"__env__ {testgitfs}")
assert len(p.remotes) == 1 assert len(p.remotes) == 1
p2.checkout() p2.checkout()
repo2 = p2.remotes[0] repo2 = p2.remotes[0]
@ -163,11 +171,19 @@ def test_pygit2_checkout_fetch_on_fail(pygit2_pillar_opts):
def _test_multiple_repos(opts): def _test_multiple_repos(opts):
## DGM p = _get_pillar(
## DGM opts,
## DGM "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git",
## DGM "main https://github.com/saltstack/salt-test-pillar-gitfs.git",
## DGM "branch https://github.com/saltstack/salt-test-pillar-gitfs.git",
## DGM "__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git",
## DGM "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git",
## DGM )
p = _get_pillar( p = _get_pillar(
opts, opts,
"__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", f"__env__ {testgitfs}",
"main https://github.com/saltstack/salt-test-pillar-gitfs.git", f"main {testgitfs}",
"branch https://github.com/saltstack/salt-test-pillar-gitfs.git", f"branch {testgitfs}",
"__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git", "__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git",
"other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git",
) )
@ -177,11 +193,19 @@ def _test_multiple_repos(opts):
assert len({r.get_cachedir() for r in p.remotes}) == 5 assert len({r.get_cachedir() for r in p.remotes}) == 5
assert len({r.get_salt_working_dir() for r in p.remotes}) == 5 assert len({r.get_salt_working_dir() for r in p.remotes}) == 5
## DGM p2 = _get_pillar(
## DGM opts,
## DGM "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git",
## DGM "main https://github.com/saltstack/salt-test-pillar-gitfs.git",
## DGM "branch https://github.com/saltstack/salt-test-pillar-gitfs.git",
## DGM "__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git",
## DGM "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git",
## DGM )
p2 = _get_pillar( p2 = _get_pillar(
opts, opts,
"__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", f"__env__ {testgitfs}",
"main https://github.com/saltstack/salt-test-pillar-gitfs.git", f"main {testgitfs}",
"branch https://github.com/saltstack/salt-test-pillar-gitfs.git", f"branch {testgitfs}",
"__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git", "__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git",
"other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git",
) )
@ -192,11 +216,19 @@ def _test_multiple_repos(opts):
assert repo.get_cachedir() == repo2.get_cachedir() assert repo.get_cachedir() == repo2.get_cachedir()
assert repo.get_salt_working_dir() == repo2.get_salt_working_dir() assert repo.get_salt_working_dir() == repo2.get_salt_working_dir()
opts["pillarenv"] = "main" opts["pillarenv"] = "main"
## DGM p3 = _get_pillar(
## DGM opts,
## DGM "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git",
## DGM "main https://github.com/saltstack/salt-test-pillar-gitfs.git",
## DGM "branch https://github.com/saltstack/salt-test-pillar-gitfs.git",
## DGM "__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git",
## DGM "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git",
## DGM )
p3 = _get_pillar( p3 = _get_pillar(
opts, opts,
"__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", f"__env__ {testgitfs}",
"main https://github.com/saltstack/salt-test-pillar-gitfs.git", f"main {testgitfs}",
"branch https://github.com/saltstack/salt-test-pillar-gitfs.git", f"branch {testgitfs}",
"__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git", "__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git",
"other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git",
) )
@ -225,17 +257,23 @@ def test_pygit2_multiple_repos(pygit2_pillar_opts):
def _test_fetch_request(opts): def _test_fetch_request(opts):
## DGM p = _get_pillar(
## DGM opts,
## DGM "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git",
## DGM "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git",
## DGM )
p = _get_pillar( p = _get_pillar(
opts, opts,
"__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", f"__env__ {testgitfs}",
"other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git",
) )
frequest = os.path.join(p.remotes[0].get_salt_working_dir(), "fetch_request") frequest = os.path.join(p.remotes[0].get_salt_working_dir(), "fetch_request")
frequest_other = os.path.join(p.remotes[1].get_salt_working_dir(), "fetch_request") frequest_other = os.path.join(p.remotes[1].get_salt_working_dir(), "fetch_request")
opts["pillarenv"] = "main" opts["pillarenv"] = "main"
p2 = _get_pillar( ## DGM p2 = _get_pillar(
opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git" ## DGM opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git"
) ## DGM )
p2 = _get_pillar(opts, f"__env__ {testgitfs}")
frequest2 = os.path.join(p2.remotes[0].get_salt_working_dir(), "fetch_request") frequest2 = os.path.join(p2.remotes[0].get_salt_working_dir(), "fetch_request")
assert frequest != frequest2 assert frequest != frequest2
assert os.path.isfile(frequest) is False assert os.path.isfile(frequest) is False
@ -275,17 +313,23 @@ def test_pygit2_fetch_request(pygit2_pillar_opts):
def _test_clear_old_remotes(opts): def _test_clear_old_remotes(opts):
## DGM p = _get_pillar(
## DGM opts,
## DGM "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git",
## DGM "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git",
## DGM )
p = _get_pillar( p = _get_pillar(
opts, opts,
"__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", f"__env__ {testgitfs}",
"other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git",
) )
repo = p.remotes[0] repo = p.remotes[0]
repo2 = p.remotes[1] repo2 = p.remotes[1]
opts["pillarenv"] = "main" opts["pillarenv"] = "main"
p2 = _get_pillar( ## DGM p2 = _get_pillar(
opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git" ## DGM opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git"
) ## DGM )
p2 = _get_pillar(opts, f"__env__ {testgitfs}")
repo3 = p2.remotes[0] repo3 = p2.remotes[0]
assert os.path.isdir(repo.get_cachedir()) is True assert os.path.isdir(repo.get_cachedir()) is True
assert os.path.isdir(repo2.get_cachedir()) is True assert os.path.isdir(repo2.get_cachedir()) is True
@ -311,9 +355,13 @@ def test_pygit2_clear_old_remotes(pygit2_pillar_opts):
def _test_remote_map(opts): def _test_remote_map(opts):
## DGM p = _get_pillar(
## DGM opts,
## DGM "https://github.com/saltstack/salt-test-pillar-gitfs.git",
## DGM )
p = _get_pillar( p = _get_pillar(
opts, opts,
"https://github.com/saltstack/salt-test-pillar-gitfs.git", testgitfs,
) )
p.fetch_remotes() p.fetch_remotes()
assert len(p.remotes) == 1 assert len(p.remotes) == 1
@ -333,9 +381,13 @@ def test_pygit2_remote_map(pygit2_pillar_opts):
def _test_lock(opts): def _test_lock(opts):
## DGM p = _get_pillar(
## DGM opts,
## DGM "https://github.com/saltstack/salt-test-pillar-gitfs.git",
## DGM )
p = _get_pillar( p = _get_pillar(
opts, opts,
"https://github.com/saltstack/salt-test-pillar-gitfs.git", testgitfs,
) )
p.fetch_remotes() p.fetch_remotes()
assert len(p.remotes) == 1 assert len(p.remotes) == 1
@ -355,8 +407,9 @@ def _test_lock(opts):
assert repo.clear_lock() == ( assert repo.clear_lock() == (
[ [
( (
f"Removed update lock for git_pillar remote " ## DGM f"Set update lock for git_pillar remote "
f"'https://github.com/saltstack/salt-test-pillar-gitfs.git' on machine_id '{mach_id}'" ## DGM f"'https://github.com/saltstack/salt-test-pillar-gitfs.git' on machine_id '{mach_id}'"
f"Set update lock for git_pillar remote '{testgitfs}' on machine_id '{mach_id}'"
) )
], ],
[], [],