From b1754eeb634f4b711f8bebd13a9bc7e696d07171 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 23 Apr 2024 03:25:15 -0700 Subject: [PATCH] Regression tests for dns defined masters Adding tests to validate we check for changing dns anytime we're disconnected from the currently connected master --- tests/pytests/scenarios/dns/__init__.py | 0 tests/pytests/scenarios/dns/conftest.py | 91 ++++++++++ .../scenarios/dns/multimaster/conftest.py | 160 ++++++++++++++++++ .../scenarios/dns/multimaster/test_dns.py | 43 +++++ tests/pytests/scenarios/dns/test_dns.py | 29 ++++ .../multimaster/test_failover_master.py | 4 - 6 files changed, 323 insertions(+), 4 deletions(-) create mode 100644 tests/pytests/scenarios/dns/__init__.py create mode 100644 tests/pytests/scenarios/dns/conftest.py create mode 100644 tests/pytests/scenarios/dns/multimaster/conftest.py create mode 100644 tests/pytests/scenarios/dns/multimaster/test_dns.py create mode 100644 tests/pytests/scenarios/dns/test_dns.py diff --git a/tests/pytests/scenarios/dns/__init__.py b/tests/pytests/scenarios/dns/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/pytests/scenarios/dns/conftest.py b/tests/pytests/scenarios/dns/conftest.py new file mode 100644 index 00000000000..e872bda6885 --- /dev/null +++ b/tests/pytests/scenarios/dns/conftest.py @@ -0,0 +1,91 @@ +import logging +import pathlib +import subprocess + +import pytest + +log = logging.getLogger(__name__) + + +class HostsFile: + """ + Simple helper class for tests that need to modify /etc/hosts. + """ + + def __init__(self, path, orig_text): + self._path = path + self._orig_text = orig_text + + @property + def orig_text(self): + return self._orig_text + + def __getattr__(self, key): + if key in ["_path", "_orig_text", "orig_text"]: + return self.__getattribute__(key) + return getattr(self._path, key) + + +@pytest.fixture +def etc_hosts(): + hosts = pathlib.Path("/etc/hosts") + orig_text = hosts.read_text(encoding="utf-8") + hosts = HostsFile(hosts, orig_text) + try: + yield hosts + finally: + hosts.write_text(orig_text) + + +@pytest.fixture(scope="package") +def master(request, salt_factories): + + subprocess.check_output(["ip", "addr", "add", "172.16.0.1/32", "dev", "lo"]) + + config_defaults = { + "open_mode": True, + "transport": request.config.getoption("--transport"), + } + config_overrides = { + "interface": "0.0.0.0", + } + factory = salt_factories.salt_master_daemon( + "master", + defaults=config_defaults, + overrides=config_overrides, + extra_cli_arguments_after_first_start_failure=["--log-level=info"], + ) + with factory.started(start_timeout=180): + yield factory + + try: + subprocess.check_output(["ip", "addr", "del", "172.16.0.1/32", "dev", "lo"]) + except subprocess.CalledProcessError: + pass + + +@pytest.fixture(scope="package") +def salt_cli(master): + return master.salt_cli(timeout=180) + + +@pytest.fixture(scope="package") +def minion(master): + config_defaults = { + "transport": master.config["transport"], + } + port = master.config["ret_port"] + config_overrides = { + "master": f"master.local:{port}", + "publish_port": master.config["publish_port"], + "master_alive_interval": 5, + "master_tries": -1, + "auth_safemode": False, + } + factory = master.salt_minion_daemon( + "minion", + defaults=config_defaults, + overrides=config_overrides, + extra_cli_arguments_after_first_start_failure=["--log-level=info"], + ) + return factory diff --git a/tests/pytests/scenarios/dns/multimaster/conftest.py b/tests/pytests/scenarios/dns/multimaster/conftest.py new file mode 100644 index 00000000000..6b948557dbe --- /dev/null +++ b/tests/pytests/scenarios/dns/multimaster/conftest.py @@ -0,0 +1,160 @@ +import logging +import os +import shutil +import subprocess + +import pytest + +log = logging.getLogger(__name__) + + +@pytest.fixture(scope="package") +def salt_mm_master_1(request, salt_factories): + + subprocess.check_output(["ip", "addr", "add", "172.16.0.1/32", "dev", "lo"]) + + config_defaults = { + "open_mode": True, + "transport": request.config.getoption("--transport"), + } + config_overrides = { + "interface": "0.0.0.0", + "master_sign_pubkey": True, + } + factory = salt_factories.salt_master_daemon( + "mm-master-1", + defaults=config_defaults, + overrides=config_overrides, + extra_cli_arguments_after_first_start_failure=["--log-level=info"], + ) + try: + with factory.started(start_timeout=180): + yield factory + finally: + + try: + subprocess.check_output(["ip", "addr", "del", "172.16.0.1/32", "dev", "lo"]) + except subprocess.CalledProcessError: + pass + + +@pytest.fixture(scope="package") +def mm_master_1_salt_cli(salt_mm_master_1): + return salt_mm_master_1.salt_cli(timeout=180) + + +@pytest.fixture(scope="package") +def salt_mm_master_2(salt_factories, salt_mm_master_1): + # if salt.utils.platform.is_darwin() or salt.utils.platform.is_freebsd(): + # subprocess.check_output(["ifconfig", "lo0", "alias", "127.0.0.2", "up"]) + + config_defaults = { + "open_mode": True, + "transport": salt_mm_master_1.config["transport"], + } + config_overrides = { + "interface": "0.0.0.0", + "master_sign_pubkey": True, + } + + # Use the same ports for both masters, they are binding to different interfaces + for key in ( + "ret_port", + "publish_port", + ): + config_overrides[key] = salt_mm_master_1.config[key] + 1 + factory = salt_factories.salt_master_daemon( + "mm-master-2", + defaults=config_defaults, + overrides=config_overrides, + extra_cli_arguments_after_first_start_failure=["--log-level=info"], + ) + + # Both masters will share the same signing key pair + for keyfile in ("master_sign.pem", "master_sign.pub"): + shutil.copyfile( + os.path.join(salt_mm_master_1.config["pki_dir"], keyfile), + os.path.join(factory.config["pki_dir"], keyfile), + ) + with factory.started(start_timeout=180): + yield factory + + +@pytest.fixture(scope="package") +def mm_master_2_salt_cli(salt_mm_master_2): + return salt_mm_master_2.salt_cli(timeout=180) + + +@pytest.fixture(scope="package") +def salt_mm_minion_1(salt_mm_master_1, salt_mm_master_2): + config_defaults = { + "transport": salt_mm_master_1.config["transport"], + } + + mm_master_1_port = salt_mm_master_1.config["ret_port"] + # mm_master_1_addr = salt_mm_master_1.config["interface"] + mm_master_2_port = salt_mm_master_2.config["ret_port"] + # mm_master_2_addr = salt_mm_master_2.config["interface"] + config_overrides = { + "master": [ + f"master1.local:{mm_master_1_port}", + f"master2.local:{mm_master_2_port}", + ], + "publish_port": salt_mm_master_1.config["publish_port"], + # "master_type": "failover", + "master_alive_interval": 5, + "master_tries": -1, + "verify_master_pubkey_sign": True, + "retry_dns": 1, + } + factory = salt_mm_master_1.salt_minion_daemon( + "mm-minion-1", + defaults=config_defaults, + overrides=config_overrides, + extra_cli_arguments_after_first_start_failure=["--log-level=info"], + ) + # Need to grab the public signing key from the master, either will do + shutil.copyfile( + os.path.join(salt_mm_master_1.config["pki_dir"], "master_sign.pub"), + os.path.join(factory.config["pki_dir"], "master_sign.pub"), + ) + # with factory.started(start_timeout=180): + yield factory + + +@pytest.fixture(scope="package") +def salt_mm_minion_2(salt_mm_master_1, salt_mm_master_2): + config_defaults = { + "transport": salt_mm_master_1.config["transport"], + } + + mm_master_1_port = salt_mm_master_1.config["ret_port"] + mm_master_1_addr = salt_mm_master_1.config["interface"] + mm_master_2_port = salt_mm_master_2.config["ret_port"] + mm_master_2_addr = salt_mm_master_2.config["interface"] + # We put the second master first in the list so it has the right startup checks every time. + config_overrides = { + "master": [ + f"{mm_master_2_addr}:{mm_master_2_port}", + f"{mm_master_1_addr}:{mm_master_1_port}", + ], + "publish_port": salt_mm_master_1.config["publish_port"], + "master_type": "failover", + "master_alive_interval": 5, + "master_tries": -1, + "verify_master_pubkey_sign": True, + "retry_dns": 1, + } + factory = salt_mm_master_2.salt_minion_daemon( + "mm-failover-minion-2", + defaults=config_defaults, + overrides=config_overrides, + extra_cli_arguments_after_first_start_failure=["--log-level=info"], + ) + # Need to grab the public signing key from the master, either will do + shutil.copyfile( + os.path.join(salt_mm_master_1.config["pki_dir"], "master_sign.pub"), + os.path.join(factory.config["pki_dir"], "master_sign.pub"), + ) + # with factory.started(start_timeout=180): + yield factory diff --git a/tests/pytests/scenarios/dns/multimaster/test_dns.py b/tests/pytests/scenarios/dns/multimaster/test_dns.py new file mode 100644 index 00000000000..4cc9db80718 --- /dev/null +++ b/tests/pytests/scenarios/dns/multimaster/test_dns.py @@ -0,0 +1,43 @@ +import logging +import subprocess +import time + +import pytest + +log = logging.getLogger(__name__) + + +@pytest.mark.skip_unless_on_linux +def test_multimaster_dns( + salt_mm_master_1, salt_mm_minion_1, mm_master_1_salt_cli, etc_hosts, caplog +): + """ + Verify a minion configured with multimaster hot/hot will pick up a master's + dns change if it's been disconnected. + """ + + etc_hosts.write_text( + f"{etc_hosts.orig_text}\n172.16.0.1 master1.local master2.local" + ) + + log.info("Added hosts record for master1.local and master2.local") + + with salt_mm_minion_1.started(start_timeout=180): + with caplog.at_level(logging.INFO): + ret = mm_master_1_salt_cli.run("test.ping", minion_tgt="mm-minion-1") + assert ret.returncode == 0 + log.info("Removing secondary master IP address.") + etc_hosts.write_text( + f"{etc_hosts.orig_text}\n127.0.0.1 master1.local master2.local" + ) + subprocess.check_output(["ip", "addr", "del", "172.16.0.1/32", "dev", "lo"]) + log.info("Changed hosts record for master1.local and master2.local") + time.sleep(15) + assert ( + "Master ip address changed from 172.16.0.1 to 127.0.0.1" in caplog.text + ) + ret = mm_master_1_salt_cli.run("test.ping", minion_tgt="mm-minion-1") + assert ret.returncode == 0 + assert ( + "Master ip address changed from 172.16.0.1 to 127.0.0.1" in caplog.text + ) diff --git a/tests/pytests/scenarios/dns/test_dns.py b/tests/pytests/scenarios/dns/test_dns.py new file mode 100644 index 00000000000..467fb7c51f1 --- /dev/null +++ b/tests/pytests/scenarios/dns/test_dns.py @@ -0,0 +1,29 @@ +import logging +import subprocess +import time + +import pytest + +log = logging.getLogger(__name__) + + +@pytest.mark.skip_unless_on_linux +def test_dns_change(master, minion, salt_cli, etc_hosts, caplog): + """ + Verify a minion will pick up a master's dns change if it's been disconnected. + """ + + etc_hosts.write_text(f"{etc_hosts.orig_text}\n172.16.0.1 master.local") + + with minion.started(start_timeout=180): + with caplog.at_level(logging.INFO): + ret = salt_cli.run("test.ping", minion_tgt="minion") + assert ret.returncode == 0 + etc_hosts.write_text(f"{etc_hosts.orig_text}\n127.0.0.1 master.local") + subprocess.check_output(["ip", "addr", "del", "172.16.0.1/32", "dev", "lo"]) + time.sleep(15) + assert ( + "Master ip address changed from 172.16.0.1 to 127.0.0.1" in caplog.text + ) + ret = salt_cli.run("test.ping", minion_tgt="minion") + assert ret.returncode == 0 diff --git a/tests/pytests/scenarios/failover/multimaster/test_failover_master.py b/tests/pytests/scenarios/failover/multimaster/test_failover_master.py index f661e9ab9a4..84ab7548ff4 100644 --- a/tests/pytests/scenarios/failover/multimaster/test_failover_master.py +++ b/tests/pytests/scenarios/failover/multimaster/test_failover_master.py @@ -159,10 +159,6 @@ def test_minions_alive_with_no_master( """ Make sure the minions stay alive after all masters have stopped. """ - if grains["os_family"] == "Debian" and grains["osmajorrelease"] == 9: - pytest.skip( - "Skipping on Debian 9 until flaky issues resolved. See issue #61749" - ) start_time = time.time() with salt_mm_failover_master_1.stopped(): with salt_mm_failover_master_2.stopped():