saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Run both the old and new versions in docker for direct comparisons

Browse source
This commit is contained in:
MKLeb 2022-12-06 21:13:01 -05:00 • committed by Megan Wilhite
parent dc3a1bd012
commit 92bdf93b60
1 changed files with 165 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

249
tests/pytests/scenarios/performance/test_performance.py
View file

@ -8,7 +8,7 @@ from saltfactories.daemons import master
from saltfactories.daemons.container import SaltDaemon, SaltMinion
from saltfactories.utils import random_string
from salt.version import SaltVersionsInfo
from salt.version import SaltVersionsInfo, __version__
pytestmark = [pytest.mark.skip_if_binaries_missing("docker")]
@ -34,7 +34,7 @@ class ContainerMinion(SaltMinion):
return []
# ---------------------- Docker Setup ----------------------
# ---------------------- Previous Version Setup ----------------------
@pytest.fixture(scope="package")
@ -43,21 +43,21 @@ def prev_version():
@pytest.fixture(scope="package")
def container_master_id(prev_version):
return random_string("master-performance-{}-".format(prev_version), uppercase=False)
def prev_master_id():
return random_string("master-performance-prev-", uppercase=False)
@pytest.fixture(scope="package")
def container_master(
def prev_master(
request,
salt_factories,
host_docker_network_ip_address,
network,
prev_version,
docker_client,
container_master_id,
prev_master_id,
):
root_dir = salt_factories.get_root_dir_for_daemon(container_master_id)
root_dir = salt_factories.get_root_dir_for_daemon(prev_master_id)
conf_dir = root_dir / "conf"
conf_dir.mkdir(exist_ok=True)
@ -79,7 +79,7 @@ def container_master(
}
factory = salt_factories.salt_master_daemon(
container_master_id,
prev_master_id,
defaults=config_defaults,
overrides=config_overrides,
factory_class=ContainerMaster,
@ -87,10 +87,10 @@ def container_master(
base_script_args=["--log-level=debug"],
container_run_kwargs={
"network": network,
"hostname": container_master_id,
"hostname": prev_master_id,
},
docker_client=docker_client,
name=container_master_id,
name=prev_master_id,
start_timeout=120,
max_start_attempts=1,
skip_if_docker_client_not_connectable=True,
@ -100,49 +100,49 @@ def container_master(
@pytest.fixture(scope="package")
def container_salt_cli(container_master):
return container_master.salt_cli()
def prev_salt_cli(prev_master):
return prev_master.salt_cli()
@pytest.fixture(scope="package")
def container_salt_key_cli(container_master):
return container_master.salt_key_cli()
def prev_salt_key_cli(prev_master):
return prev_master.salt_key_cli()
@pytest.fixture(scope="package")
def container_salt_run_cli(container_master):
return container_master.salt_run_cli()
def prev_salt_run_cli(prev_master):
return prev_master.salt_run_cli()
@pytest.fixture(scope="package")
def container_minion_id(prev_version):
def prev_minion_id():
return random_string(
"minion-performance-{}-".format(prev_version),
"minion-performance-prev-",
uppercase=False,
)
@pytest.fixture(scope="package")
def container_minion(
container_minion_id,
container_master,
def prev_minion(
prev_minion_id,
prev_master,
docker_client,
prev_version,
host_docker_network_ip_address,
network,
container_master_id,
prev_master_id,
):
config_overrides = {
"master": container_master_id,
"master": prev_master_id,
"user": False,
"pytest-minion": {"log": {"host": host_docker_network_ip_address}},
}
factory = container_master.salt_minion_daemon(
container_minion_id,
factory = prev_master.salt_minion_daemon(
prev_minion_id,
overrides=config_overrides,
factory_class=ContainerMinion,
# SaltMinion kwargs
name=container_minion_id,
name=prev_minion_id,
image="ghcr.io/saltstack/salt-ci-containers/salt:{}".format(prev_version),
docker_client=docker_client,
start_timeout=120,
@ -150,21 +150,21 @@ def container_minion(
skip_if_docker_client_not_connectable=True,
container_run_kwargs={
"network": network,
"hostname": container_minion_id,
"hostname": prev_minion_id,
},
max_start_attempts=1,
)
factory.after_terminate(
pytest.helpers.remove_stale_minion_key, container_master, factory.id
pytest.helpers.remove_stale_minion_key, prev_master, factory.id
)
with factory.started():
yield factory
@pytest.fixture
def container_sls(sls_contents, state_tree, tmp_path):
sls_name = "container"
location = tmp_path / "container" / "testfile"
def prev_sls(sls_contents, state_tree, tmp_path):
sls_name = "prev"
location = tmp_path / "prev" / "testfile"
location.parent.mkdir()
with pytest.helpers.temp_file(
"{}.sls".format(sls_name), sls_contents.format(path=str(location)), state_tree
@ -172,21 +172,29 @@ def container_sls(sls_contents, state_tree, tmp_path):
yield sls_name
# ---------------------- Local Setup ----------------------
# ---------------------- Current Version Setup ----------------------
def _install_local_salt(factory):
factory.run("pip install /saltcode")
@pytest.fixture(scope="package")
def master_id():
def curr_master_id():
return random_string("master-performance-", uppercase=False)
@pytest.fixture(scope="package")
def salt_master(
def curr_master(
request,
salt_factories,
master_id,
host_docker_network_ip_address,
network,
prev_version,
docker_client,
curr_master_id,
):
root_dir = salt_factories.get_root_dir_for_daemon(master_id)
root_dir = salt_factories.get_root_dir_for_daemon(curr_master_id)
conf_dir = root_dir / "conf"
conf_dir.mkdir(exist_ok=True)
@ -198,59 +206,113 @@ def salt_master(
publish_port = ports.get_unused_localhost_port()
ret_port = ports.get_unused_localhost_port()
config_overrides = {
"interface": "127.0.0.1",
"interface": "0.0.0.0",
"publish_port": publish_port,
"ret_port": ret_port,
"log_level_logfile": "quiet",
"pytest-master": {
"log": {"host": host_docker_network_ip_address},
},
}
factory = salt_factories.salt_master_daemon(
master_id,
curr_master_id,
defaults=config_defaults,
overrides=config_overrides,
factory_class=ContainerMaster,
image="ghcr.io/saltstack/salt-ci-containers/salt:{}".format(prev_version),
base_script_args=["--log-level=debug"],
container_run_kwargs={
"network": network,
"hostname": curr_master_id,
# Bind the current code to a directory for pip installing
"volumes": {
os.environ["REPO_ROOT_DIR"]: {"bind": "/saltcode", "mode": "z"}
},
},
docker_client=docker_client,
name=curr_master_id,
start_timeout=120,
max_start_attempts=1,
skip_if_docker_client_not_connectable=True,
)
factory.before_start(_install_local_salt, factory)
with factory.started():
yield factory
@pytest.fixture(scope="package")
def salt_cli(salt_master):
return salt_master.salt_cli()
def curr_salt_cli(curr_master):
return curr_master.salt_cli()
@pytest.fixture(scope="package")
def minion_id():
def curr_salt_run_cli(curr_master):
return curr_master.salt_run_cli()
@pytest.fixture(scope="package")
def curr_salt_key_cli(curr_master):
return curr_master.salt_key_cli()
@pytest.fixture(scope="package")
def curr_minion_id():
return random_string(
"minion-performance-",
"minion-performance-curr-",
uppercase=False,
)
@pytest.fixture(scope="package")
def salt_minion(
minion_id,
salt_master,
def curr_minion(
curr_minion_id,
curr_master,
docker_client,
prev_version,
host_docker_network_ip_address,
network,
curr_master_id,
):
config_overrides = {
"master": curr_master_id,
"user": False,
"pytest-minion": {"log": {"host": host_docker_network_ip_address}},
}
factory = salt_master.salt_minion_daemon(
minion_id,
factory = curr_master.salt_minion_daemon(
curr_minion_id,
overrides=config_overrides,
factory_class=ContainerMinion,
# SaltMinion kwargs
name=curr_minion_id,
image="ghcr.io/saltstack/salt-ci-containers/salt:{}".format(prev_version),
docker_client=docker_client,
start_timeout=120,
pull_before_start=False,
skip_if_docker_client_not_connectable=True,
container_run_kwargs={
"network": network,
"hostname": curr_minion_id,
# Bind the current code to a directory for pip installing
"volumes": {
os.environ["REPO_ROOT_DIR"]: {"bind": "/saltcode", "mode": "z"}
},
},
max_start_attempts=1,
)
factory.before_start(_install_local_salt, factory)
factory.after_terminate(
pytest.helpers.remove_stale_minion_key, salt_master, factory.id
pytest.helpers.remove_stale_minion_key, curr_master, factory.id
)
with factory.started():
yield factory
@pytest.fixture
def local_sls(sls_contents, state_tree, tmp_path):
sls_name = "local"
location = tmp_path / "local" / "testfile"
def curr_sls(sls_contents, state_tree, tmp_path):
sls_name = "curr"
location = tmp_path / "curr" / "testfile"
location.parent.mkdir()
with pytest.helpers.temp_file(
"{}.sls".format(sls_name), sls_contents.format(path=str(location)), state_tree
@ -262,6 +324,7 @@ def _wait_for_stdout(expected, func, *args, timeout=120, **kwargs):
start = time.time()
while time.time() < start + timeout:
ret = func(*args, **kwargs)
print(ret)
if ret and ret.stdout and expected in ret.stdout:
break
time.sleep(1)
@ -269,62 +332,80 @@ def _wait_for_stdout(expected, func, *args, timeout=120, **kwargs):
pytest.skip("Skipping test, one or more daemons failed to start")
# @pytest.mark.flaky(max_runs=4)
def test_performance(
container_salt_cli,
container_minion,
container_salt_run_cli,
container_salt_key_cli,
prev_salt_cli,
prev_minion,
prev_salt_run_cli,
prev_salt_key_cli,
prev_version,
container_master,
prev_master,
state_tree,
salt_cli,
salt_minion,
salt_master,
container_sls,
local_sls,
curr_salt_cli,
curr_master,
curr_salt_run_cli,
curr_salt_key_cli,
curr_minion,
prev_sls,
curr_sls,
):
# Copy all of the needed files to both master file roots directories
shutil.copytree(
state_tree, salt_master.config["file_roots"]["base"][0], dirs_exist_ok=True
state_tree, curr_master.config["file_roots"]["base"][0], dirs_exist_ok=True
)
shutil.copytree(
state_tree, container_master.config["file_roots"]["base"][0], dirs_exist_ok=True
state_tree, prev_master.config["file_roots"]["base"][0], dirs_exist_ok=True
)
# Wait for the container master and minion to start
# Wait for the old master and minion to start
_wait_for_stdout(
prev_version, container_master.run, *container_salt_run_cli.cmdline("--version")
prev_version, prev_master.run, *prev_salt_run_cli.cmdline("--version")
)
salt_key_cmd = [
comp
for comp in container_salt_key_cli.cmdline("-Ay")
for comp in prev_salt_key_cli.cmdline("-Ay")
if not comp.startswith("--log-level")
]
_wait_for_stdout(container_minion.id, container_master.run, *salt_key_cmd)
_wait_for_stdout(prev_minion.id, prev_master.run, *salt_key_cmd)
_wait_for_stdout(
"Salt: {}".format(prev_version),
container_master.run,
*container_salt_cli.cmdline("test.versions", minion_tgt=container_minion.id)
prev_master.run,
*prev_salt_cli.cmdline("test.versions", minion_tgt=prev_minion.id)
)
# Wait for the new master and minion to start
_wait_for_stdout("3005", curr_master.run, *curr_salt_run_cli.cmdline("--version"))
curr_key_cmd = [
comp
for comp in curr_salt_key_cli.cmdline("-Ay")
if not comp.startswith("--log-level")
]
_wait_for_stdout(curr_minion.id, curr_master.run, *curr_key_cmd)
_wait_for_stdout(
"Salt: {}".format("3005"),
curr_master.run,
*curr_salt_cli.cmdline("test.versions", minion_tgt=curr_minion.id)
)
# Let's now apply the states
applies = os.environ.get("SALT_PERFORMANCE_TEST_APPLIES", 3)
container_salt_cmd = container_salt_cli.cmdline(
"state.apply", container_sls, minion_tgt=container_minion.id
)
start = time.time()
prev_duration = 0
for _ in range(applies):
container_state_ret = container_master.run(*container_salt_cmd)
assert container_state_ret.data
container_duration = time.time() - start
start = time.time()
for _ in range(applies):
local_state_ret = salt_cli.run(
"state.apply", local_sls, minion_tgt=salt_minion.id
prev_state_ret = prev_master.run(
*prev_salt_cli.cmdline("state.apply", prev_sls, minion_tgt=prev_minion.id)
)
assert local_state_ret.data
local_duration = time.time() - start
assert prev_state_ret.data
for _, ret in prev_state_ret.data[prev_minion.id].items():
prev_duration += ret["duration"]
assert local_duration <= 1.01 * container_duration
curr_duration = 0
for _ in range(applies):
curr_state_ret = curr_master.run(
*curr_salt_cli.cmdline("state.apply", curr_sls, minion_tgt=curr_minion.id)
)
assert curr_state_ret.data
for _, ret in curr_state_ret.data[curr_minion.id].items():
curr_duration += ret["duration"]
assert curr_duration <= 1.05 * prev_duration