Reduce duplication, de-clutter, simplify

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>
This commit is contained in:
Pedro Algarvio 2023-12-06 13:23:36 +00:00 committed by Daniel Wozniak
parent a9b8193d1e
commit 59f4904e29
14 changed files with 162 additions and 777 deletions

View file

@ -12,15 +12,8 @@ from saltfactories.utils import random_string
from saltfactories.utils.tempfiles import SaltPillarTree, SaltStateTree
import salt.config
from tests.pytests.pkg.support.helpers import (
CODE_DIR,
TESTS_DIR,
ApiRequest,
SaltMaster,
SaltMasterWindows,
SaltPkgInstall,
TestUser,
)
from tests.conftest import CODE_DIR, TESTS_DIR
from tests.support.pkg import ApiRequest, SaltMaster, SaltMasterWindows, SaltPkgInstall
log = logging.getLogger(__name__)
@ -303,7 +296,9 @@ def sls(state_tree):
@pytest.fixture(scope="session")
def salt_master(salt_factories, install_salt, state_tree, pillar_tree):
def salt_master(
salt_factories, install_salt, state_tree, pillar_tree, pkg_tests_account
):
"""
Start up a master
"""
@ -327,7 +322,13 @@ def salt_master(salt_factories, install_salt, state_tree, pillar_tree):
"pillar_roots": pillar_tree.as_dict(),
"rest_cherrypy": {"port": 8000, "disable_ssl": True},
"netapi_enable_clients": ["local"],
"external_auth": {"auto": {"saltdev": [".*"]}},
"external_auth": {
"auto": {
pkg_tests_account.username: [
".*",
],
},
},
"fips_mode": FIPS_TESTRUN,
"open_mode": True,
}
@ -520,9 +521,9 @@ def salt_call_cli(salt_minion):
return salt_minion.salt_call_cli()
@pytest.fixture(scope="module")
def test_account(salt_call_cli):
with TestUser(salt_call_cli=salt_call_cli) as account:
@pytest.fixture(scope="session")
def pkg_tests_account():
with pytest.helpers.create_account() as account:
yield account
@ -557,6 +558,8 @@ def salt_api(salt_master, install_salt, extras_pypath):
@pytest.fixture(scope="module")
def api_request(test_account, salt_api):
with ApiRequest(salt_api=salt_api, test_account=test_account) as session:
def api_request(pkg_tests_account, salt_api):
with ApiRequest(
port=salt_api.config["rest_cherrypy"]["port"], account=pkg_tests_account
) as session:
yield session

View file

@ -1,53 +0,0 @@
#!py
import importlib
def run():
config = {}
for test_import in [
'templates', 'platform', 'cli', 'executors', 'config', 'wheel', 'netapi',
'cache', 'proxy', 'transport', 'metaproxy', 'modules', 'tokens', 'matchers',
'acl', 'auth', 'log', 'engines', 'client', 'returners', 'runners', 'tops',
'output', 'daemons', 'thorium', 'renderers', 'states', 'cloud', 'roster',
'beacons', 'pillar', 'spm', 'utils', 'sdb', 'fileserver', 'defaults',
'ext', 'queues', 'grains', 'serializers'
]:
try:
import_name = "salt.{}".format(test_import)
importlib.import_module(import_name)
config['test_imports_succeeded'] = {
'test.succeed_without_changes': [
{
'name': import_name
},
],
}
except ModuleNotFoundError as err:
config['test_imports_failed'] = {
'test.fail_without_changes': [
{
'name': import_name,
'comment': "The imports test failed. The error was: {}".format(err)
},
],
}
for stdlib_import in ["telnetlib"]:
try:
importlib.import_module(stdlib_import)
config['stdlib_imports_succeeded'] = {
'test.succeed_without_changes': [
{
'name': stdlib_import
},
],
}
except ModuleNotFoundError as err:
config['stdlib_imports_failed'] = {
'test.fail_without_changes': [
{
'name': stdlib_import,
'comment': "The stdlib imports test failed. The error was: {}".format(err)
},
],
}
return config

View file

@ -1,13 +0,0 @@
import sys
import salt.utils.data
user_arg = sys.argv
if user_arg[1] == "raise":
raise Exception("test")
if salt.utils.data.is_true(user_arg[1]):
sys.exit(0)
else:
sys.exit(1)

View file

@ -10,11 +10,77 @@ pytestmark = [
log = logging.getLogger(__name__)
def test_check_imports(salt_cli, salt_minion):
CHECK_IMPORTS_SLS_CONTENTS = """
#!py
import importlib
def run():
config = {}
for test_import in [
'templates', 'platform', 'cli', 'executors', 'config', 'wheel', 'netapi',
'cache', 'proxy', 'transport', 'metaproxy', 'modules', 'tokens', 'matchers',
'acl', 'auth', 'log', 'engines', 'client', 'returners', 'runners', 'tops',
'output', 'daemons', 'thorium', 'renderers', 'states', 'cloud', 'roster',
'beacons', 'pillar', 'spm', 'utils', 'sdb', 'fileserver', 'defaults',
'ext', 'queues', 'grains', 'serializers'
]:
try:
import_name = "salt.{}".format(test_import)
importlib.import_module(import_name)
config['test_imports_succeeded'] = {
'test.succeed_without_changes': [
{
'name': import_name
},
],
}
except ModuleNotFoundError as err:
config['test_imports_failed'] = {
'test.fail_without_changes': [
{
'name': import_name,
'comment': "The imports test failed. The error was: {}".format(err)
},
],
}
for stdlib_import in ["telnetlib"]:
try:
importlib.import_module(stdlib_import)
config['stdlib_imports_succeeded'] = {
'test.succeed_without_changes': [
{
'name': stdlib_import
},
],
}
except ModuleNotFoundError as err:
config['stdlib_imports_failed'] = {
'test.fail_without_changes': [
{
'name': stdlib_import,
'comment': "The stdlib imports test failed. The error was: {}".format(err)
},
],
}
return config
"""
@pytest.fixture
def state_name(salt_master):
name = "check-imports"
with salt_master.state_tree.base.temp_file(
f"{name}.sls", CHECK_IMPORTS_SLS_CONTENTS
):
yield name
def test_check_imports(salt_cli, salt_minion, state_name):
"""
Test imports
"""
ret = salt_cli.run("state.sls", "check_imports", minion_tgt=salt_minion.id)
ret = salt_cli.run("state.sls", state_name, minion_tgt=salt_minion.id)
assert ret.returncode == 0
assert ret.data
result = MultiStateResult(raw=ret.data)

View file

@ -42,6 +42,14 @@ def wipe_pydeps(shell, install_salt, extras_pypath):
shutil.rmtree(dirname, ignore_errors=True)
@pytest.fixture
def pkg_tests_account_environ(pkg_tests_account):
environ = os.environ.copy()
environ["LOGNAME"] = environ["USER"] = pkg_tests_account.username
environ["HOME"] = pkg_tests_account.info.home
return environ
def test_pip_install(salt_call_cli, install_salt, shell):
"""
Test pip.install and ensure module can use installed library
@ -98,18 +106,25 @@ def test_pip_install_extras(shell, install_salt, extras_pypath_bin):
assert ret.returncode == 0
def demote(user_uid, user_gid):
def demote(account):
def result():
# os.setgid does not remove group membership, so we remove them here so they are REALLY non-root
os.setgroups([])
os.setgid(user_gid)
os.setuid(user_uid)
os.setgid(account.info.gid)
os.setuid(account.info.uid)
return result
@pytest.mark.skip_on_windows(reason="We can't easily demote users on Windows")
def test_pip_non_root(shell, install_salt, test_account, extras_pypath_bin, pypath):
def test_pip_non_root(
shell,
install_salt,
pkg_tests_account,
extras_pypath_bin,
pypath,
pkg_tests_account_environ,
):
if install_salt.classic:
pytest.skip("We can install non-root for classic packages")
check_path = extras_pypath_bin / "pep8"
@ -118,8 +133,8 @@ def test_pip_non_root(shell, install_salt, test_account, extras_pypath_bin, pypa
# We should be able to issue a --help without being root
ret = subprocess.run(
install_salt.binary_paths["salt"] + ["--help"],
preexec_fn=demote(test_account.uid, test_account.gid),
env=test_account.env,
preexec_fn=demote(pkg_tests_account),
env=pkg_tests_account_environ,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=False,
@ -141,8 +156,8 @@ def test_pip_non_root(shell, install_salt, test_account, extras_pypath_bin, pypa
# Now, we should still not be able to install as non-root
ret = subprocess.run(
install_salt.binary_paths["pip"] + ["install", "pep8"],
preexec_fn=demote(test_account.uid, test_account.gid),
env=test_account.env,
preexec_fn=demote(pkg_tests_account),
env=pkg_tests_account_environ,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=False,

View file

@ -1,9 +1,8 @@
import subprocess
import textwrap
import pytest
from tests.pytests.pkg.support.helpers import TESTS_DIR
@pytest.fixture
def python_script_bin(install_salt):
@ -13,13 +12,40 @@ def python_script_bin(install_salt):
return install_salt.binary_paths["python"]
@pytest.fixture
def check_python_file(tmp_path):
script_path = tmp_path / "check_python.py"
script_path.write_text(
textwrap.dedent(
"""
import sys
import salt.utils.data
user_arg = sys.argv
if user_arg[1] == "raise":
raise Exception("test")
if salt.utils.data.is_true(user_arg[1]):
sys.exit(0)
else:
sys.exit(1)
"""
)
)
return script_path
@pytest.mark.parametrize("exp_ret,user_arg", [(1, "false"), (0, "true")])
def test_python_script(install_salt, exp_ret, user_arg, python_script_bin):
def test_python_script(
install_salt, exp_ret, user_arg, python_script_bin, check_python_file
):
ret = install_salt.proc.run(
*(
python_script_bin
+ [
str(TESTS_DIR / "pytests" / "pkg" / "files" / "check_python.py"),
str(check_python_file),
user_arg,
]
),
@ -32,12 +58,12 @@ def test_python_script(install_salt, exp_ret, user_arg, python_script_bin):
assert ret.returncode == exp_ret, ret.stderr
def test_python_script_exception(install_salt, python_script_bin):
def test_python_script_exception(install_salt, python_script_bin, check_python_file):
ret = install_salt.proc.run(
*(
python_script_bin
+ [
str(TESTS_DIR / "pytests" / "pkg" / "files" / "check_python.py"),
str(check_python_file),
"raise",
]
),

View file

@ -49,11 +49,13 @@ def test_salt_call_local_sys_doc_aliases(salt_call_cli):
@pytest.mark.skip_on_windows()
def test_salt_call_cmd_run_id_runas(salt_call_cli, test_account, caplog):
def test_salt_call_cmd_run_id_runas(salt_call_cli, pkg_tests_account, caplog):
"""
Test salt-call --local cmd_run id with runas
"""
ret = salt_call_cli.run("--local", "cmd.run", "id", runas=test_account.username)
ret = salt_call_cli.run(
"--local", "cmd.run", "id", runas=pkg_tests_account.username
)
assert "Environment could not be retrieved for user" not in caplog.text
assert str(test_account.uid) in ret.stdout
assert str(test_account.gid) in ret.stdout
assert str(pkg_tests_account.info.uid) in ret.stdout
assert str(pkg_tests_account.info.gid) in ret.stdout

View file

@ -175,7 +175,7 @@ def test_pkg_paths(
@pytest.mark.skip_if_binaries_missing("logrotate")
def test_paths_log_rotation(
salt_master, salt_minion, salt_call_cli, install_salt, test_account
salt_master, salt_minion, salt_call_cli, install_salt, pkg_tests_account
):
"""
Test the correct ownership is assigned when log rotation occurs
@ -267,7 +267,7 @@ def test_paths_log_rotation(
"file.replace",
f"{install_salt.conf_dir}/master",
"user: salt",
f"user: {test_account.username}",
f"user: {pkg_tests_account.username}",
"flags=['IGNORECASE']",
"append_if_not_found=True",
)
@ -276,7 +276,7 @@ def test_paths_log_rotation(
# change ownership of appropriate paths to user
for _path in log_pkg_paths:
chg_ownership_cmd = (
f"chown -R {test_account.username} {_path}"
f"chown -R {pkg_tests_account.username} {_path}"
)
ret = salt_call_cli.run(
"--local", "cmd.run", chg_ownership_cmd
@ -317,7 +317,9 @@ def test_paths_log_rotation(
for _path in log_files_list:
log_path = pathlib.Path(_path)
assert log_path.exists()
assert log_path.owner() == test_account.username
assert (
log_path.owner() == pkg_tests_account.username
)
assert log_path.stat().st_mode & 0o7777 == 0o640
# cleanup
@ -328,7 +330,7 @@ def test_paths_log_rotation(
"--local",
"file.replace",
f"{install_salt.conf_dir}/master",
f"user: {test_account.username}",
f"user: {pkg_tests_account.username}",
"user: salt",
"flags=['IGNORECASE']",
"append_if_not_found=True",

View file

@ -1,11 +0,0 @@
"""
Python will always try to import sitecustomize.
We use that fact to try and support code coverage for sub-processes
"""
try:
import coverage
coverage.process_startup()
except ImportError:
pass

View file

@ -1,102 +0,0 @@
"""
:codeauthor: Pedro Algarvio (pedro@algarvio.me)
:copyright: Copyright 2017 by the SaltStack Team, see AUTHORS for more details.
:license: Apache 2.0, see LICENSE for more details.
tests.support.paths
~~~~~~~~~~~~~~~~~~~
Tests related paths
"""
import logging
import os
import re
import sys
import tempfile
log = logging.getLogger(__name__)
SALT_CODE_DIR = os.path.join(
os.path.dirname(
os.path.dirname(
os.path.dirname(
os.path.dirname(os.path.normpath(os.path.abspath(__file__)))
)
)
),
"salt",
)
TESTS_DIR = os.path.join(os.path.dirname(SALT_CODE_DIR), "tests")
if TESTS_DIR.startswith("//"):
# Have we been given an initial double forward slash? Ditch it!
TESTS_DIR = TESTS_DIR[1:]
if sys.platform.startswith("win"):
TESTS_DIR = os.path.normcase(TESTS_DIR)
CODE_DIR = os.path.dirname(TESTS_DIR)
if sys.platform.startswith("win"):
CODE_DIR = CODE_DIR.replace("\\", "\\\\")
UNIT_TEST_DIR = os.path.join(TESTS_DIR, "unit")
INTEGRATION_TEST_DIR = os.path.join(TESTS_DIR, "integration")
# Let's inject CODE_DIR so salt is importable if not there already
if TESTS_DIR in sys.path:
sys.path.remove(TESTS_DIR)
if CODE_DIR in sys.path and sys.path[0] != CODE_DIR:
sys.path.remove(CODE_DIR)
if CODE_DIR not in sys.path:
sys.path.insert(0, CODE_DIR)
if TESTS_DIR not in sys.path:
sys.path.insert(1, TESTS_DIR)
SYS_TMP_DIR = os.path.abspath(
os.path.realpath(
# Avoid ${TMPDIR} and gettempdir() on MacOS as they yield a base path too long
# for unix sockets: ``error: AF_UNIX path too long``
# Gentoo Portage prefers ebuild tests are rooted in ${TMPDIR}
os.environ.get("TMPDIR", tempfile.gettempdir())
if not sys.platform.startswith("darwin")
else "/tmp"
)
)
TMP = os.path.join(SYS_TMP_DIR, "salt-tests-tmpdir")
TMP_ROOT_DIR = os.path.join(TMP, "rootdir")
FILES = os.path.join(INTEGRATION_TEST_DIR, "files")
BASE_FILES = os.path.join(INTEGRATION_TEST_DIR, "files", "file", "base")
PROD_FILES = os.path.join(INTEGRATION_TEST_DIR, "files", "file", "prod")
PYEXEC = "python{}.{}".format(*sys.version_info)
MOCKBIN = os.path.join(INTEGRATION_TEST_DIR, "mockbin")
SCRIPT_DIR = os.path.join(CODE_DIR, "scripts")
TMP_STATE_TREE = os.path.join(SYS_TMP_DIR, "salt-temp-state-tree")
TMP_PILLAR_TREE = os.path.join(SYS_TMP_DIR, "salt-temp-pillar-tree")
TMP_PRODENV_STATE_TREE = os.path.join(SYS_TMP_DIR, "salt-temp-prodenv-state-tree")
TMP_PRODENV_PILLAR_TREE = os.path.join(SYS_TMP_DIR, "salt-temp-prodenv-pillar-tree")
TMP_CONF_DIR = TMP_MINION_CONF_DIR = os.path.join(TMP, "config")
TMP_SUB_MINION_CONF_DIR = os.path.join(TMP_CONF_DIR, "sub-minion")
TMP_SYNDIC_MINION_CONF_DIR = os.path.join(TMP_CONF_DIR, "syndic-minion")
TMP_SYNDIC_MASTER_CONF_DIR = os.path.join(TMP_CONF_DIR, "syndic-master")
TMP_SSH_CONF_DIR = TMP_MINION_CONF_DIR
CONF_DIR = os.path.join(INTEGRATION_TEST_DIR, "files", "conf")
PILLAR_DIR = os.path.join(FILES, "pillar")
TMP_SCRIPT_DIR = os.path.join(TMP, "scripts")
ENGINES_DIR = os.path.join(FILES, "engines")
LOG_HANDLERS_DIR = os.path.join(FILES, "log_handlers")
def list_test_mods():
"""
A generator which returns all of the test files
"""
test_re = re.compile(r"^test_.+\.py$")
for dirname in (UNIT_TEST_DIR, INTEGRATION_TEST_DIR):
test_type = os.path.basename(dirname)
for root, _, files in os.walk(dirname):
parent_mod = root[len(dirname) :].lstrip(os.sep).replace(os.sep, ".")
for filename in files:
if test_re.match(filename):
mod_name = test_type
if parent_mod:
mod_name += "." + parent_mod
mod_name += "." + filename[:-3]
yield mod_name

View file

@ -1,209 +0,0 @@
"""
:codeauthor: Pedro Algarvio (pedro@algarvio.me)
.. _runtime_vars:
Runtime Variables
-----------------
:command:`salt-runtests` provides a variable, :py:attr:`RUNTIME_VARS` which has some common paths defined at
startup:
.. autoattribute:: tests.support.runtests.RUNTIME_VARS
:annotation:
:TMP: Tests suite temporary directory
:TMP_CONF_DIR: Configuration directory from where the daemons that :command:`salt-runtests` starts get their
configuration files.
:TMP_CONF_MASTER_INCLUDES: Salt Master configuration files includes directory. See
:salt_conf_master:`default_include`.
:TMP_CONF_MINION_INCLUDES: Salt Minion configuration files includes directory. Seei
:salt_conf_minion:`include`.
:TMP_CONF_CLOUD_INCLUDES: Salt cloud configuration files includes directory. The same as the salt master and
minion includes configuration, though under a different directory name.
:TMP_CONF_CLOUD_PROFILE_INCLUDES: Salt cloud profiles configuration files includes directory. Same as above.
:TMP_CONF_CLOUD_PROVIDER_INCLUDES: Salt cloud providers configuration files includes directory. Same as above.
:TMP_SCRIPT_DIR: Temporary scripts directory from where the Salt CLI tools will be called when running tests.
:TMP_SALT_INTEGRATION_FILES: Temporary directory from where Salt's test suite integration files are copied to.
:TMP_BASEENV_STATE_TREE: Salt master's **base** environment state tree directory
:TMP_PRODENV_STATE_TREE: Salt master's **production** environment state tree directory
:TMP_BASEENV_PILLAR_TREE: Salt master's **base** environment pillar tree directory
:TMP_PRODENV_PILLAR_TREE: Salt master's **production** environment pillar tree directory
Use it on your test case in case of need. As simple as:
.. code-block:: python
import os
from tests.support.runtests import RUNTIME_VARS
# Path to the testing minion configuration file
minion_config_path = os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'minion')
.. _`pytest`: http://pytest.org
"""
import logging
import os
import shutil
import salt.utils.path
import salt.utils.platform
import tests.support.paths as paths
try:
import pwd
except ImportError:
import salt.utils.win_functions
log = logging.getLogger(__name__)
def this_user():
"""
Get the user associated with the current process.
"""
if salt.utils.platform.is_windows():
return salt.utils.win_functions.get_current_user(with_domain=False)
return pwd.getpwuid(os.getuid())[0]
class RootsDict(dict):
def merge(self, data):
for key, values in data.items():
if key not in self:
self[key] = values
continue
for value in values:
if value not in self[key]:
self[key].append(value)
return self
def to_dict(self):
return dict(self)
def recursive_copytree(source, destination, overwrite=False):
for root, dirs, files in os.walk(source):
for item in dirs:
src_path = os.path.join(root, item)
dst_path = os.path.join(
destination, src_path.replace(source, "").lstrip(os.sep)
)
if not os.path.exists(dst_path):
log.debug("Creating directory: %s", dst_path)
os.makedirs(dst_path)
for item in files:
src_path = os.path.join(root, item)
dst_path = os.path.join(
destination, src_path.replace(source, "").lstrip(os.sep)
)
if os.path.exists(dst_path) and not overwrite:
if os.stat(src_path).st_mtime > os.stat(dst_path).st_mtime:
log.debug("Copying %s to %s", src_path, dst_path)
shutil.copy2(src_path, dst_path)
else:
if not os.path.isdir(os.path.dirname(dst_path)):
log.debug("Creating directory: %s", os.path.dirname(dst_path))
os.makedirs(os.path.dirname(dst_path))
log.debug("Copying %s to %s", src_path, dst_path)
shutil.copy2(src_path, dst_path)
class RuntimeVars:
__self_attributes__ = ("_vars", "_locked", "lock")
def __init__(self, **kwargs):
self._vars = kwargs
self._locked = False
def lock(self):
# Late import
from salt.utils.immutabletypes import freeze
frozen_vars = freeze(self._vars.copy())
self._vars = frozen_vars
self._locked = True
def __iter__(self):
yield from self._vars.items()
def __getattribute__(self, name):
if name in object.__getattribute__(self, "_vars"):
return object.__getattribute__(self, "_vars")[name]
return object.__getattribute__(self, name)
def __setattr__(self, name, value):
if getattr(self, "_locked", False) is True:
raise RuntimeError(
"After {} is locked, no additional data can be added to it".format(
self.__class__.__name__
)
)
if name in object.__getattribute__(self, "__self_attributes__"):
object.__setattr__(self, name, value)
return
self._vars[name] = value
# <---- Helper Methods -----------------------------------------------------------------------------------------------
# ----- Global Variables -------------------------------------------------------------------------------------------->
XML_OUTPUT_DIR = os.environ.get(
"SALT_XML_TEST_REPORTS_DIR", os.path.join(paths.TMP, "xml-test-reports")
)
# <---- Global Variables ---------------------------------------------------------------------------------------------
# ----- Tests Runtime Variables ------------------------------------------------------------------------------------->
RUNTIME_VARS = RuntimeVars(
TMP=paths.TMP,
SYS_TMP_DIR=paths.SYS_TMP_DIR,
FILES=paths.FILES,
CONF_DIR=paths.CONF_DIR,
PILLAR_DIR=paths.PILLAR_DIR,
ENGINES_DIR=paths.ENGINES_DIR,
LOG_HANDLERS_DIR=paths.LOG_HANDLERS_DIR,
TMP_ROOT_DIR=paths.TMP_ROOT_DIR,
TMP_CONF_DIR=paths.TMP_CONF_DIR,
TMP_MINION_CONF_DIR=paths.TMP_MINION_CONF_DIR,
TMP_CONF_MASTER_INCLUDES=os.path.join(paths.TMP_CONF_DIR, "master.d"),
TMP_CONF_MINION_INCLUDES=os.path.join(paths.TMP_CONF_DIR, "minion.d"),
TMP_CONF_PROXY_INCLUDES=os.path.join(paths.TMP_CONF_DIR, "proxy.d"),
TMP_CONF_CLOUD_INCLUDES=os.path.join(paths.TMP_CONF_DIR, "cloud.conf.d"),
TMP_CONF_CLOUD_PROFILE_INCLUDES=os.path.join(
paths.TMP_CONF_DIR, "cloud.profiles.d"
),
TMP_CONF_CLOUD_PROVIDER_INCLUDES=os.path.join(
paths.TMP_CONF_DIR, "cloud.providers.d"
),
TMP_SUB_MINION_CONF_DIR=paths.TMP_SUB_MINION_CONF_DIR,
TMP_SYNDIC_MASTER_CONF_DIR=paths.TMP_SYNDIC_MASTER_CONF_DIR,
TMP_SYNDIC_MINION_CONF_DIR=paths.TMP_SYNDIC_MINION_CONF_DIR,
TMP_SSH_CONF_DIR=paths.TMP_SSH_CONF_DIR,
TMP_SCRIPT_DIR=paths.TMP_SCRIPT_DIR,
TMP_STATE_TREE=paths.TMP_STATE_TREE,
TMP_BASEENV_STATE_TREE=paths.TMP_STATE_TREE,
TMP_PILLAR_TREE=paths.TMP_PILLAR_TREE,
TMP_BASEENV_PILLAR_TREE=paths.TMP_PILLAR_TREE,
TMP_PRODENV_STATE_TREE=paths.TMP_PRODENV_STATE_TREE,
TMP_PRODENV_PILLAR_TREE=paths.TMP_PRODENV_PILLAR_TREE,
SHELL_TRUE_PATH=salt.utils.path.which("true")
if not salt.utils.platform.is_windows()
else "cmd /c exit 0 > nul",
SHELL_FALSE_PATH=salt.utils.path.which("false")
if not salt.utils.platform.is_windows()
else "cmd /c exit 1 > nul",
RUNNING_TESTS_USER=this_user(),
RUNTIME_CONFIGS={},
CODE_DIR=paths.CODE_DIR,
SALT_CODE_DIR=paths.SALT_CODE_DIR,
BASE_FILES=paths.BASE_FILES,
PROD_FILES=paths.PROD_FILES,
TESTS_DIR=paths.TESTS_DIR,
)
# <---- Tests Runtime Variables --------------------------------------------------------------------------------------

View file

@ -1,256 +0,0 @@
"""
tests.support.sminion
~~~~~~~~~~~~~~~~~~~~~
SMinion's support functions
"""
import fnmatch
import hashlib
import logging
import os
import shutil
import sys
import salt.minion
import salt.utils.path
import salt.utils.stringutils
from tests.support.runtests import RUNTIME_VARS
log = logging.getLogger(__name__)
DEFAULT_SMINION_ID = "pytest-internal-sminion"
def build_minion_opts(
minion_id=None,
root_dir=None,
initial_conf_file=None,
minion_opts_overrides=None,
skip_cached_opts=False,
cache_opts=True,
minion_role=None,
):
if minion_id is None:
minion_id = DEFAULT_SMINION_ID
if skip_cached_opts is False:
try:
opts_cache = build_minion_opts.__cached_opts__
except AttributeError:
opts_cache = build_minion_opts.__cached_opts__ = {}
cached_opts = opts_cache.get(minion_id)
if cached_opts:
return cached_opts
log.info("Generating testing minion %r configuration...", minion_id)
if root_dir is None:
hashed_minion_id = hashlib.sha1()
hashed_minion_id.update(salt.utils.stringutils.to_bytes(minion_id))
root_dir = os.path.join(
RUNTIME_VARS.TMP_ROOT_DIR, hashed_minion_id.hexdigest()[:6]
)
if initial_conf_file is not None:
minion_opts = salt.config._read_conf_file(
initial_conf_file
) # pylint: disable=protected-access
else:
minion_opts = {}
conf_dir = os.path.join(root_dir, "conf")
conf_file = os.path.join(conf_dir, "minion")
minion_opts["id"] = minion_id
minion_opts["conf_file"] = conf_file
minion_opts["root_dir"] = root_dir
minion_opts["cachedir"] = "cache"
minion_opts["user"] = RUNTIME_VARS.RUNNING_TESTS_USER
minion_opts["pki_dir"] = "pki"
minion_opts["hosts.file"] = os.path.join(RUNTIME_VARS.TMP_ROOT_DIR, "hosts")
minion_opts["aliases.file"] = os.path.join(RUNTIME_VARS.TMP_ROOT_DIR, "aliases")
minion_opts["file_client"] = "local"
minion_opts["server_id_use_crc"] = "adler32"
minion_opts["pillar_roots"] = {"base": [RUNTIME_VARS.TMP_PILLAR_TREE]}
minion_opts["file_roots"] = {
"base": [
# Let's support runtime created files that can be used like:
# salt://my-temp-file.txt
RUNTIME_VARS.TMP_STATE_TREE
],
# Alternate root to test __env__ choices
"prod": [
os.path.join(RUNTIME_VARS.FILES, "file", "prod"),
RUNTIME_VARS.TMP_PRODENV_STATE_TREE,
],
}
if initial_conf_file and initial_conf_file.startswith(RUNTIME_VARS.FILES):
# We assume we were passed a minion configuration file defined fo testing and, as such
# we define the file and pillar roots to include the testing states/pillar trees
minion_opts["pillar_roots"]["base"].append(
os.path.join(RUNTIME_VARS.FILES, "pillar", "base"),
)
minion_opts["file_roots"]["base"].append(
os.path.join(RUNTIME_VARS.FILES, "file", "base"),
)
minion_opts["file_roots"]["prod"].append(
os.path.join(RUNTIME_VARS.FILES, "file", "prod"),
)
# We need to copy the extension modules into the new master root_dir or
# it will be prefixed by it
extension_modules_path = os.path.join(root_dir, "extension_modules")
if not os.path.exists(extension_modules_path):
shutil.copytree(
os.path.join(RUNTIME_VARS.FILES, "extension_modules"),
extension_modules_path,
)
minion_opts["extension_modules"] = extension_modules_path
# Custom grains
if "grains" not in minion_opts:
minion_opts["grains"] = {}
if minion_role is not None:
minion_opts["grains"]["role"] = minion_role
# Under windows we can't seem to properly create a virtualenv off of another
# virtualenv, we can on linux but we will still point to the virtualenv binary
# outside the virtualenv running the test suite, if that's the case.
try:
real_prefix = sys.real_prefix
# The above attribute exists, this is a virtualenv
if salt.utils.platform.is_windows():
virtualenv_binary = os.path.join(real_prefix, "Scripts", "virtualenv.exe")
else:
# We need to remove the virtualenv from PATH or we'll get the virtualenv binary
# from within the virtualenv, we don't want that
path = os.environ.get("PATH")
if path is not None:
path_items = path.split(os.pathsep)
for item in path_items[:]:
if item.startswith(sys.base_prefix):
path_items.remove(item)
os.environ["PATH"] = os.pathsep.join(path_items)
virtualenv_binary = salt.utils.path.which("virtualenv")
if path is not None:
# Restore previous environ PATH
os.environ["PATH"] = path
if not virtualenv_binary.startswith(real_prefix):
virtualenv_binary = None
if virtualenv_binary and not os.path.exists(virtualenv_binary):
# It doesn't exist?!
virtualenv_binary = None
except AttributeError:
# We're not running inside a virtualenv
virtualenv_binary = None
if virtualenv_binary:
minion_opts["venv_bin"] = virtualenv_binary
# Override minion_opts with minion_opts_overrides
if minion_opts_overrides:
minion_opts.update(minion_opts_overrides)
if not os.path.exists(conf_dir):
os.makedirs(conf_dir)
with salt.utils.files.fopen(conf_file, "w") as fp_:
salt.utils.yaml.safe_dump(minion_opts, fp_, default_flow_style=False)
log.info("Generating testing minion %r configuration completed.", minion_id)
minion_opts = salt.config.minion_config(
conf_file, minion_id=minion_id, cache_minion_id=True
)
salt.utils.verify.verify_env(
[
os.path.join(minion_opts["pki_dir"], "accepted"),
os.path.join(minion_opts["pki_dir"], "rejected"),
os.path.join(minion_opts["pki_dir"], "pending"),
os.path.dirname(minion_opts["log_file"]),
minion_opts["extension_modules"],
minion_opts["cachedir"],
minion_opts["sock_dir"],
RUNTIME_VARS.TMP_STATE_TREE,
RUNTIME_VARS.TMP_PILLAR_TREE,
RUNTIME_VARS.TMP_PRODENV_STATE_TREE,
RUNTIME_VARS.TMP,
],
RUNTIME_VARS.RUNNING_TESTS_USER,
root_dir=root_dir,
)
if cache_opts:
try:
opts_cache = build_minion_opts.__cached_opts__
except AttributeError:
opts_cache = build_minion_opts.__cached_opts__ = {}
opts_cache[minion_id] = minion_opts
return minion_opts
def create_sminion(
minion_id=None,
root_dir=None,
initial_conf_file=None,
sminion_cls=salt.minion.SMinion,
minion_opts_overrides=None,
skip_cached_minion=False,
cache_sminion=True,
):
if minion_id is None:
minion_id = DEFAULT_SMINION_ID
if skip_cached_minion is False:
try:
minions_cache = create_sminion.__cached_minions__
except AttributeError:
create_sminion.__cached_minions__ = {}
cached_minion = create_sminion.__cached_minions__.get(minion_id)
if cached_minion:
return cached_minion
minion_opts = build_minion_opts(
minion_id=minion_id,
root_dir=root_dir,
initial_conf_file=initial_conf_file,
minion_opts_overrides=minion_opts_overrides,
skip_cached_opts=skip_cached_minion,
cache_opts=cache_sminion,
)
log.info("Instantiating a testing %s(%s)", sminion_cls.__name__, minion_id)
sminion = sminion_cls(minion_opts)
if cache_sminion:
try:
minions_cache = create_sminion.__cached_minions__
except AttributeError:
minions_cache = create_sminion.__cached_minions__ = {}
minions_cache[minion_id] = sminion
return sminion
def check_required_sminion_attributes(sminion_attr, required_items):
"""
:param sminion_attr: The name of the sminion attribute to check, such as 'functions' or 'states'
:param required_items: The items that must be part of the designated sminion attribute for the decorated test
:return The packages that are not available
"""
required_salt_items = set(required_items)
sminion = create_sminion(minion_id=DEFAULT_SMINION_ID)
available_items = list(getattr(sminion, sminion_attr))
not_available_items = set()
name = "__not_available_{items}s__".format(items=sminion_attr)
if not hasattr(sminion, name):
setattr(sminion, name, set())
cached_not_available_items = getattr(sminion, name)
for not_available_item in cached_not_available_items:
if not_available_item in required_salt_items:
not_available_items.add(not_available_item)
required_salt_items.remove(not_available_item)
for required_item_name in required_salt_items:
search_name = required_item_name
if "." not in search_name:
search_name += ".*"
if not fnmatch.filter(available_items, search_name):
not_available_items.add(required_item_name)
cached_not_available_items.add(required_item_name)
return not_available_items

View file

@ -30,22 +30,9 @@ from saltfactories.daemons import api, master, minion
from saltfactories.utils import cli_scripts
import salt.utils.files
from tests.conftest import CODE_DIR
from tests.support.pytest.helpers import TestAccount
try:
import crypt
HAS_CRYPT = True
except ImportError:
HAS_CRYPT = False
try:
import pwd
HAS_PWD = True
except ImportError:
HAS_PWD = False
TESTS_DIR = pathlib.Path(__file__).resolve().parent.parent.parent.parent
CODE_DIR = TESTS_DIR.parent
ARTIFACTS_DIR = CODE_DIR / "artifacts" / "pkg"
log = logging.getLogger(__name__)
@ -1451,82 +1438,10 @@ class SaltKey(PkgMixin, key.SaltKey):
key.SaltKey.__attrs_post_init__(self)
@attr.s(kw_only=True, slots=True)
class TestUser:
"""
Add a test user
"""
salt_call_cli = attr.ib()
username = attr.ib(default="saltdev")
# Must follow Windows Password Complexity requirements
password = attr.ib(default="P@ssW0rd")
_pw_record = attr.ib(init=False, repr=False, default=None)
def salt_call_local(self, *args):
ret = self.salt_call_cli.run("--local", *args)
if ret.returncode != 0:
log.error(ret)
assert ret.returncode == 0
return ret.data
def add_user(self):
log.debug("Adding system account %r", self.username)
if platform.is_windows():
self.salt_call_local("user.add", self.username, self.password)
else:
self.salt_call_local("user.add", self.username)
hash_passwd = crypt.crypt(self.password, crypt.mksalt(crypt.METHOD_SHA512))
self.salt_call_local("shadow.set_password", self.username, hash_passwd)
assert self.username in self.salt_call_local("user.list_users")
def remove_user(self):
log.debug("Removing system account %r", self.username)
if platform.is_windows():
self.salt_call_local(
"user.delete", self.username, "purge=True", "force=True"
)
else:
self.salt_call_local("user.delete", self.username, "remove=True")
@property
def pw_record(self):
if self._pw_record is None and HAS_PWD:
self._pw_record = pwd.getpwnam(self.username)
return self._pw_record
@property
def uid(self):
if HAS_PWD:
return self.pw_record.pw_uid
return None
@property
def gid(self):
if HAS_PWD:
return self.pw_record.pw_gid
return None
@property
def env(self):
environ = os.environ.copy()
environ["LOGNAME"] = environ["USER"] = self.username
environ["HOME"] = self.pw_record.pw_dir
return environ
def __enter__(self):
self.add_user()
return self
def __exit__(self, *_):
self.remove_user()
@attr.s(kw_only=True, slots=True)
class ApiRequest:
salt_api: SaltApi = attr.ib(repr=False)
test_account: TestUser = attr.ib(repr=False)
port: int = attr.ib(repr=False)
account: TestAccount = attr.ib(repr=False)
session: requests.Session = attr.ib(init=False, repr=False)
api_uri: str = attr.ib(init=False)
auth_data: Dict[str, str] = attr.ib(init=False)
@ -1537,13 +1452,13 @@ class ApiRequest:
@api_uri.default
def _default_api_uri(self):
return f"http://localhost:{self.salt_api.config['rest_cherrypy']['port']}"
return f"http://localhost:{self.port}"
@auth_data.default
def _default_auth_data(self):
return {
"username": self.test_account.username,
"password": self.test_account.password,
"username": self.account.username,
"password": self.account.password,
"eauth": "auto",
"out": "json",
}