From 3d2c111f5805ee3515c3b9fd8e935185851b9297 Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Fri, 16 Feb 2024 16:15:29 -0700 Subject: [PATCH] [WIP] Almost working port from unit test to pytest, contains debugging --- tests/pytests/unit/utils/test_gitfs_locks.py | 333 ++++++++++++++++++- 1 file changed, 324 insertions(+), 9 deletions(-) diff --git a/tests/pytests/unit/utils/test_gitfs_locks.py b/tests/pytests/unit/utils/test_gitfs_locks.py index c5260da3d9f..0b9b814af48 100644 --- a/tests/pytests/unit/utils/test_gitfs_locks.py +++ b/tests/pytests/unit/utils/test_gitfs_locks.py @@ -3,7 +3,14 @@ These only test the provider selection and verification logic, they do not init any remotes. """ +import logging +import pathlib + +## import os +import tempfile + import pytest +from saltfactories.utils import random_string import salt.ext.tornado.ioloop import salt.fileserver.gitfs @@ -11,7 +18,70 @@ import salt.utils.files import salt.utils.gitfs import salt.utils.path import salt.utils.platform -from tests.support.mixins import AdaptedConfigurationTestCaseMixin +from salt.utils.immutabletypes import freeze +from salt.utils.verify import verify_env + +## from tests.support.mixins import AdaptedConfigurationTestCaseMixin +from tests.support.runtests import RUNTIME_VARS + +log = logging.getLogger(__name__) + + +@pytest.fixture(scope="session", autouse=True) +def _create_old_tempdir(): + pathlib.Path(RUNTIME_VARS.TMP).mkdir(exist_ok=True, parents=True) + + +@pytest.fixture(scope="session", autouse=True) +def bridge_pytest_and_runtests( + reap_stray_processes, + salt_factories, + salt_syndic_master_factory, + salt_syndic_factory, + salt_master_factory, + salt_minion_factory, + salt_sub_minion_factory, + sshd_config_dir, +): + # Make sure unittest2 uses the pytest generated configuration + RUNTIME_VARS.RUNTIME_CONFIGS["master"] = freeze(salt_master_factory.config) + RUNTIME_VARS.RUNTIME_CONFIGS["minion"] = freeze(salt_minion_factory.config) + RUNTIME_VARS.RUNTIME_CONFIGS["sub_minion"] = freeze(salt_sub_minion_factory.config) + RUNTIME_VARS.RUNTIME_CONFIGS["syndic_master"] = freeze( + salt_syndic_master_factory.config + ) + RUNTIME_VARS.RUNTIME_CONFIGS["syndic"] = freeze(salt_syndic_factory.config) + RUNTIME_VARS.RUNTIME_CONFIGS["client_config"] = freeze( + salt.config.client_config(salt_master_factory.config["conf_file"]) + ) + + # Make sure unittest2 classes know their paths + RUNTIME_VARS.TMP_ROOT_DIR = str(salt_factories.root_dir.resolve()) + RUNTIME_VARS.TMP_CONF_DIR = pathlib.PurePath( + salt_master_factory.config["conf_file"] + ).parent + RUNTIME_VARS.TMP_MINION_CONF_DIR = pathlib.PurePath( + salt_minion_factory.config["conf_file"] + ).parent + RUNTIME_VARS.TMP_SUB_MINION_CONF_DIR = pathlib.PurePath( + salt_sub_minion_factory.config["conf_file"] + ).parent + RUNTIME_VARS.TMP_SYNDIC_MASTER_CONF_DIR = pathlib.PurePath( + salt_syndic_master_factory.config["conf_file"] + ).parent + RUNTIME_VARS.TMP_SYNDIC_MINION_CONF_DIR = pathlib.PurePath( + salt_syndic_factory.config["conf_file"] + ).parent + RUNTIME_VARS.TMP_SSH_CONF_DIR = str(sshd_config_dir) + + +## @pytest.fixture(scope="session", autouse=True) +## def get_tmp_dir(tmp_path): +## dirpath = tmp_path / "git_test" +## dirpath.mkdir(parents=True) +## return dirpath +## +## ## dirpath.cleanup() def _clear_instance_map(): @@ -23,13 +93,247 @@ def _clear_instance_map(): pass -@pytest.fixture -def get_tmp_dir(tmp_path): - dirpath = tmp_path / "git_test" - dirpath.mkdir(parents=True) - return dirpath +class AdaptedConfigurationTestCaseMixin: - ## dirpath.cleanup() + ## __slots__ = () + + @staticmethod + def get_temp_config(config_for, **config_overrides): + log.debug( + f"DGM AdaptedConfigurationTestCaseMixin get_temp_config, config_for '{config_for}, config_overrides '{config_overrides}', runtime_vars tmp '{RUNTIME_VARS.TMP}', user '{RUNTIME_VARS.RUNNING_TESTS_USER}'" + ) + + rootdir = config_overrides.get("root_dir", RUNTIME_VARS.TMP) + + if not pathlib.Path(rootdir).exists(): + log.debug( + f"DGM AdaptedConfigurationTestCaseMixin get_temp_config, oddity runtime_vars '{RUNTIME_VARS.TMP}' should already exist" + ) + pathlib.Path(RUNTIME_VARS.TMP).mkdir(exist_ok=True, parents=True) + + rootdir = config_overrides.get("root_dir", RUNTIME_VARS.TMP) + ## if not os.path.exists(rootdir): + ## log.debug(f"DGM AdaptedConfigurationTestCaseMixin get_temp_config, oddity runtime_vars '{RUNTIME_VARS.TMP}' should already exist") + ## os.makedirs(rootdir) + + ## conf_dir = config_overrides.pop("conf_dir", os.path.join(rootdir, "conf")) + conf_dir = config_overrides.pop( + "conf_dir", pathlib.PurePath(rootdir).joinpath("conf") + ) + for key in ("cachedir", "pki_dir", "sock_dir"): + if key not in config_overrides: + config_overrides[key] = key + if "log_file" not in config_overrides: + config_overrides["log_file"] = f"logs/{config_for}.log".format() + if "user" not in config_overrides: + config_overrides["user"] = RUNTIME_VARS.RUNNING_TESTS_USER + config_overrides["root_dir"] = rootdir + + cdict = AdaptedConfigurationTestCaseMixin.get_config( + config_for, from_scratch=True + ) + + if config_for in ("master", "client_config"): + rdict = salt.config.apply_master_config(config_overrides, cdict) + if config_for == "minion": + minion_id = ( + config_overrides.get("id") + or config_overrides.get("minion_id") + or cdict.get("id") + or cdict.get("minion_id") + or random_string("temp-minion-") + ) + config_overrides["minion_id"] = config_overrides["id"] = minion_id + rdict = salt.config.apply_minion_config( + config_overrides, cdict, cache_minion_id=False, minion_id=minion_id + ) + + ## verify_env( + ## [ + ## os.path.join(rdict["pki_dir"], "minions"), + ## os.path.join(rdict["pki_dir"], "minions_pre"), + ## os.path.join(rdict["pki_dir"], "minions_rejected"), + ## os.path.join(rdict["pki_dir"], "minions_denied"), + ## os.path.join(rdict["cachedir"], "jobs"), + ## os.path.join(rdict["cachedir"], "tokens"), + ## os.path.join(rdict["root_dir"], "cache", "tokens"), + ## os.path.join(rdict["pki_dir"], "accepted"), + ## os.path.join(rdict["pki_dir"], "rejected"), + ## os.path.join(rdict["pki_dir"], "pending"), + ## os.path.dirname(rdict["log_file"]), + ## rdict["sock_dir"], + ## conf_dir, + ## ], + ## RUNTIME_VARS.RUNNING_TESTS_USER, + ## root_dir=rdict["root_dir"], + ## ) + verify_env( + [ + pathlib.PurePath(rdict["pki_dir"]).joinpath("minions"), + pathlib.PurePath(rdict["pki_dir"]).joinpath("minions_pre"), + pathlib.PurePath(rdict["pki_dir"]).joinpath("minions_rejected"), + pathlib.PurePath(rdict["pki_dir"]).joinpath("minions_denied"), + pathlib.PurePath(rdict["cachedir"]).joinpath("jobs"), + pathlib.PurePath(rdict["cachedir"]).joinpath("tokens"), + pathlib.PurePath(rdict["root_dir"]).joinpath("cache", "tokens"), + pathlib.PurePath(rdict["pki_dir"]).joinpath("accepted"), + pathlib.PurePath(rdict["pki_dir"]).joinpath("rejected"), + pathlib.PurePath(rdict["pki_dir"]).joinpath("pending"), + pathlib.PurePath(rdict["log_file"]).parent, + rdict["sock_dir"], + conf_dir, + ], + RUNTIME_VARS.RUNNING_TESTS_USER, + root_dir=rdict["root_dir"], + ) + + ## rdict["conf_file"] = os.path.join(conf_dir, config_for) + rdict["conf_file"] = pathlib.PurePath(conf_dir).joinpath(config_for) + with salt.utils.files.fopen(rdict["conf_file"], "w") as wfh: + salt.utils.yaml.safe_dump(rdict, wfh, default_flow_style=False) + return rdict + + @staticmethod + def get_config(config_for, from_scratch=False): + log.debug( + f"DGM AdaptedConfigurationTestCaseMixin get_config, config_for '{config_for}, from_scratch '{from_scratch}', runtime runtime_configs '{RUNTIME_VARS.RUNTIME_CONFIGS}'" + ) + + if from_scratch: + if config_for in ("master", "syndic_master", "mm_master", "mm_sub_master"): + return salt.config.master_config( + AdaptedConfigurationTestCaseMixin.get_config_file_path(config_for) + ) + elif config_for in ("minion", "sub_minion"): + return salt.config.minion_config( + AdaptedConfigurationTestCaseMixin.get_config_file_path(config_for), + cache_minion_id=False, + ) + elif config_for in ("syndic",): + return salt.config.syndic_config( + AdaptedConfigurationTestCaseMixin.get_config_file_path(config_for), + AdaptedConfigurationTestCaseMixin.get_config_file_path("minion"), + ) + elif config_for == "client_config": + return salt.config.client_config( + AdaptedConfigurationTestCaseMixin.get_config_file_path("master") + ) + + if config_for not in RUNTIME_VARS.RUNTIME_CONFIGS: + if config_for in ("master", "syndic_master", "mm_master", "mm_sub_master"): + RUNTIME_VARS.RUNTIME_CONFIGS[config_for] = freeze( + salt.config.master_config( + AdaptedConfigurationTestCaseMixin.get_config_file_path( + config_for + ) + ) + ) + elif config_for in ("minion", "sub_minion"): + RUNTIME_VARS.RUNTIME_CONFIGS[config_for] = freeze( + salt.config.minion_config( + AdaptedConfigurationTestCaseMixin.get_config_file_path( + config_for + ) + ) + ) + elif config_for in ("syndic",): + RUNTIME_VARS.RUNTIME_CONFIGS[config_for] = freeze( + salt.config.syndic_config( + AdaptedConfigurationTestCaseMixin.get_config_file_path( + config_for + ), + AdaptedConfigurationTestCaseMixin.get_config_file_path( + "minion" + ), + ) + ) + elif config_for == "client_config": + RUNTIME_VARS.RUNTIME_CONFIGS[config_for] = freeze( + salt.config.client_config( + AdaptedConfigurationTestCaseMixin.get_config_file_path("master") + ) + ) + return RUNTIME_VARS.RUNTIME_CONFIGS[config_for] + + @property + def config_dir(self): + return RUNTIME_VARS.TMP_CONF_DIR + + def get_config_dir(self): + log.warning("Use the config_dir attribute instead of calling get_config_dir()") + return self.config_dir + + @staticmethod + def get_config_file_path(filename): + ## if filename == "master": + ## return os.path.join(RUNTIME_VARS.TMP_CONF_DIR, filename) + ## if filename == "minion": + ## return os.path.join(RUNTIME_VARS.TMP_MINION_CONF_DIR, filename) + ## if filename == "syndic_master": + ## return os.path.join(RUNTIME_VARS.TMP_SYNDIC_MASTER_CONF_DIR, "master") + ## if filename == "syndic": + ## return os.path.join(RUNTIME_VARS.TMP_SYNDIC_MINION_CONF_DIR, "minion") + ## if filename == "sub_minion": + ## return os.path.join(RUNTIME_VARS.TMP_SUB_MINION_CONF_DIR, "minion") + ## if filename == "mm_master": + ## return os.path.join(RUNTIME_VARS.TMP_MM_CONF_DIR, "master") + ## if filename == "mm_sub_master": + ## return os.path.join(RUNTIME_VARS.TMP_MM_SUB_CONF_DIR, "master") + ## if filename == "mm_minion": + ## return os.path.join(RUNTIME_VARS.TMP_MM_MINION_CONF_DIR, "minion") + ## if filename == "mm_sub_minion": + ## return os.path.join(RUNTIME_VARS.TMP_MM_SUB_MINION_CONF_DIR, "minion") + ## return os.path.join(RUNTIME_VARS.TMP_CONF_DIR, filename) + if filename == "master": + return pathlib.PurePath(RUNTIME_VARS.TMP_CONF_DIR).joinpath(filename) + if filename == "minion": + return pathlib.PurePath(RUNTIME_VARS.TMP_MINION_CONF_DIR).joinpath(filename) + if filename == "syndic_master": + return pathlib.PurePath(RUNTIME_VARS.TMP_SYNDIC_MASTER_CONF_DIR).joinpath( + "master" + ) + if filename == "syndic": + return pathlib.PurePath(RUNTIME_VARS.TMP_SYNDIC_MINION_CONF_DIR).joinpath( + "minion" + ) + if filename == "sub_minion": + return pathlib.PurePath(RUNTIME_VARS.TMP_SUB_MINION_CONF_DIR).joinpath( + "minion" + ) + if filename == "mm_master": + return pathlib.PurePath(RUNTIME_VARS.TMP_MM_CONF_DIR).joinpath("master") + if filename == "mm_sub_master": + return pathlib.PurePath(RUNTIME_VARS.TMP_MM_SUB_CONF_DIR).joinpath("master") + if filename == "mm_minion": + return pathlib.PurePath(RUNTIME_VARS.TMP_MM_MINION_CONF_DIR).joinpath( + "minion" + ) + if filename == "mm_sub_minion": + return pathlib.PurePath(RUNTIME_VARS.TMP_MM_SUB_MINION_CONF_DIR).joinpath( + "minion" + ) + return pathlib.PurePath(RUNTIME_VARS.TMP_CONF_DIR).joinpath(filename) + + @property + def master_opts(self): + """ + Return the options used for the master + """ + return self.get_config("master") + + @property + def minion_opts(self): + """ + Return the options used for the minion + """ + return self.get_config("minion") + + @property + def sub_minion_opts(self): + """ + Return the options used for the sub_minion + """ + return self.get_config("sub_minion") class TestGitBase(AdaptedConfigurationTestCaseMixin): @@ -40,6 +344,12 @@ class TestGitBase(AdaptedConfigurationTestCaseMixin): def __init__( self, ): + # TBD DGM need to fixup using tmp_path fixture + ## self._tmp_dir = pathlib.Path(tmp_path / "git_test").mkdir(exist_ok=True, parents=True) + ## tmp_name = str(self._tmp_dir) + self._tmp_dir = tempfile.TemporaryDirectory() + tmp_name = self._tmp_dir.name + class MockedProvider( salt.utils.gitfs.GitProvider ): # pylint: disable=abstract-method @@ -66,7 +376,7 @@ class TestGitBase(AdaptedConfigurationTestCaseMixin): ) def init_remote(self): - self.gitdir = salt.utils.path.join(get_tmp_dir, ".git") + self.gitdir = salt.utils.path.join(tmp_name, ".git") self.repo = True new = False return new @@ -108,12 +418,17 @@ class TestGitBase(AdaptedConfigurationTestCaseMixin): @pytest.fixture def main_class(tmp_path): test_git_base = TestGitBase() - yield test_git_base + log.debug(f"DGM main_class, test_git_base '{dir(test_git_base)}'") + log.debug( + f"DGM main_class, test_git_base.main_class '{dir(test_git_base.main_class)}'" + ) + yield test_git_base.main_class test_git_base.tearDown() def test_update_all(main_class): + log.debug(f"DGM test_update_all, main_class '{dir(main_class)}'") main_class.update() assert len(main_class.remotes) == 2, "Wrong number of remotes" assert main_class.remotes[0].fetched