Regression tests for dns defined masters

Adding tests to validate we check for changing dns anytime we're
disconnected from the currently connected master

tests/pytests/scenarios/dns/conftest.py

@@ -0,0 +1,91 @@
+import logging
+import pathlib
+import subprocess
+
+import pytest
+
+log = logging.getLogger(__name__)
+
+
+class HostsFile:
+    """
+    Simple helper class for tests that need to modify /etc/hosts.
+    """
+
+    def __init__(self, path, orig_text):
+        self._path = path
+        self._orig_text = orig_text
+
+    @property
+    def orig_text(self):
+        return self._orig_text
+
+    def __getattr__(self, key):
+        if key in ["_path", "_orig_text", "orig_text"]:
+            return self.__getattribute__(key)
+        return getattr(self._path, key)
+
+
+@pytest.fixture
+def etc_hosts():
+    hosts = pathlib.Path("/etc/hosts")
+    orig_text = hosts.read_text(encoding="utf-8")
+    hosts = HostsFile(hosts, orig_text)
+    try:
+        yield hosts
+    finally:
+        hosts.write_text(orig_text)
+
+
+@pytest.fixture(scope="package")
+def master(request, salt_factories):
+
+    subprocess.check_output(["ip", "addr", "add", "172.16.0.1/32", "dev", "lo"])
+
+    config_defaults = {
+        "open_mode": True,
+        "transport": request.config.getoption("--transport"),
+    }
+    config_overrides = {
+        "interface": "0.0.0.0",
+    }
+    factory = salt_factories.salt_master_daemon(
+        "master",
+        defaults=config_defaults,
+        overrides=config_overrides,
+        extra_cli_arguments_after_first_start_failure=["--log-level=info"],
+    )
+    with factory.started(start_timeout=180):
+        yield factory
+
+    try:
+        subprocess.check_output(["ip", "addr", "del", "172.16.0.1/32", "dev", "lo"])
+    except subprocess.CalledProcessError:
+        pass
+
+
+@pytest.fixture(scope="package")
+def salt_cli(master):
+    return master.salt_cli(timeout=180)
+
+
+@pytest.fixture(scope="package")
+def minion(master):
+    config_defaults = {
+        "transport": master.config["transport"],
+    }
+    port = master.config["ret_port"]
+    config_overrides = {
+        "master": f"master.local:{port}",
+        "publish_port": master.config["publish_port"],
+        "master_alive_interval": 5,
+        "master_tries": -1,
+        "auth_safemode": False,
+    }
+    factory = master.salt_minion_daemon(
+        "minion",
+        defaults=config_defaults,
+        overrides=config_overrides,
+        extra_cli_arguments_after_first_start_failure=["--log-level=info"],
+    )
+    return factory

tests/pytests/scenarios/dns/multimaster/conftest.py

@@ -0,0 +1,160 @@
+import logging
+import os
+import shutil
+import subprocess
+
+import pytest
+
+log = logging.getLogger(__name__)
+
+
+@pytest.fixture(scope="package")
+def salt_mm_master_1(request, salt_factories):
+
+    subprocess.check_output(["ip", "addr", "add", "172.16.0.1/32", "dev", "lo"])
+
+    config_defaults = {
+        "open_mode": True,
+        "transport": request.config.getoption("--transport"),
+    }
+    config_overrides = {
+        "interface": "0.0.0.0",
+        "master_sign_pubkey": True,
+    }
+    factory = salt_factories.salt_master_daemon(
+        "mm-master-1",
+        defaults=config_defaults,
+        overrides=config_overrides,
+        extra_cli_arguments_after_first_start_failure=["--log-level=info"],
+    )
+    try:
+        with factory.started(start_timeout=180):
+            yield factory
+    finally:
+
+        try:
+            subprocess.check_output(["ip", "addr", "del", "172.16.0.1/32", "dev", "lo"])
+        except subprocess.CalledProcessError:
+            pass
+
+
+@pytest.fixture(scope="package")
+def mm_master_1_salt_cli(salt_mm_master_1):
+    return salt_mm_master_1.salt_cli(timeout=180)
+
+
+@pytest.fixture(scope="package")
+def salt_mm_master_2(salt_factories, salt_mm_master_1):
+    # if salt.utils.platform.is_darwin() or salt.utils.platform.is_freebsd():
+    #    subprocess.check_output(["ifconfig", "lo0", "alias", "127.0.0.2", "up"])
+
+    config_defaults = {
+        "open_mode": True,
+        "transport": salt_mm_master_1.config["transport"],
+    }
+    config_overrides = {
+        "interface": "0.0.0.0",
+        "master_sign_pubkey": True,
+    }
+
+    # Use the same ports for both masters, they are binding to different interfaces
+    for key in (
+        "ret_port",
+        "publish_port",
+    ):
+        config_overrides[key] = salt_mm_master_1.config[key] + 1
+    factory = salt_factories.salt_master_daemon(
+        "mm-master-2",
+        defaults=config_defaults,
+        overrides=config_overrides,
+        extra_cli_arguments_after_first_start_failure=["--log-level=info"],
+    )
+
+    # Both masters will share the same signing key pair
+    for keyfile in ("master_sign.pem", "master_sign.pub"):
+        shutil.copyfile(
+            os.path.join(salt_mm_master_1.config["pki_dir"], keyfile),
+            os.path.join(factory.config["pki_dir"], keyfile),
+        )
+    with factory.started(start_timeout=180):
+        yield factory
+
+
+@pytest.fixture(scope="package")
+def mm_master_2_salt_cli(salt_mm_master_2):
+    return salt_mm_master_2.salt_cli(timeout=180)
+
+
+@pytest.fixture(scope="package")
+def salt_mm_minion_1(salt_mm_master_1, salt_mm_master_2):
+    config_defaults = {
+        "transport": salt_mm_master_1.config["transport"],
+    }
+
+    mm_master_1_port = salt_mm_master_1.config["ret_port"]
+    # mm_master_1_addr = salt_mm_master_1.config["interface"]
+    mm_master_2_port = salt_mm_master_2.config["ret_port"]
+    # mm_master_2_addr = salt_mm_master_2.config["interface"]
+    config_overrides = {
+        "master": [
+            f"master1.local:{mm_master_1_port}",
+            f"master2.local:{mm_master_2_port}",
+        ],
+        "publish_port": salt_mm_master_1.config["publish_port"],
+        # "master_type": "failover",
+        "master_alive_interval": 5,
+        "master_tries": -1,
+        "verify_master_pubkey_sign": True,
+        "retry_dns": 1,
+    }
+    factory = salt_mm_master_1.salt_minion_daemon(
+        "mm-minion-1",
+        defaults=config_defaults,
+        overrides=config_overrides,
+        extra_cli_arguments_after_first_start_failure=["--log-level=info"],
+    )
+    # Need to grab the public signing key from the master, either will do
+    shutil.copyfile(
+        os.path.join(salt_mm_master_1.config["pki_dir"], "master_sign.pub"),
+        os.path.join(factory.config["pki_dir"], "master_sign.pub"),
+    )
+    # with factory.started(start_timeout=180):
+    yield factory
+
+
+@pytest.fixture(scope="package")
+def salt_mm_minion_2(salt_mm_master_1, salt_mm_master_2):
+    config_defaults = {
+        "transport": salt_mm_master_1.config["transport"],
+    }
+
+    mm_master_1_port = salt_mm_master_1.config["ret_port"]
+    mm_master_1_addr = salt_mm_master_1.config["interface"]
+    mm_master_2_port = salt_mm_master_2.config["ret_port"]
+    mm_master_2_addr = salt_mm_master_2.config["interface"]
+    # We put the second master first in the list so it has the right startup checks every time.
+    config_overrides = {
+        "master": [
+            f"{mm_master_2_addr}:{mm_master_2_port}",
+            f"{mm_master_1_addr}:{mm_master_1_port}",
+        ],
+        "publish_port": salt_mm_master_1.config["publish_port"],
+        "master_type": "failover",
+        "master_alive_interval": 5,
+        "master_tries": -1,
+        "verify_master_pubkey_sign": True,
+        "retry_dns": 1,
+    }
+    factory = salt_mm_master_2.salt_minion_daemon(
+        "mm-failover-minion-2",
+        defaults=config_defaults,
+        overrides=config_overrides,
+        extra_cli_arguments_after_first_start_failure=["--log-level=info"],
+    )
+    # Need to grab the public signing key from the master, either will do
+    shutil.copyfile(
+        os.path.join(salt_mm_master_1.config["pki_dir"], "master_sign.pub"),
+        os.path.join(factory.config["pki_dir"], "master_sign.pub"),
+    )
+    # with factory.started(start_timeout=180):
+    yield factory

tests/pytests/scenarios/dns/multimaster/test_dns.py

@@ -0,0 +1,43 @@
+import logging
+import subprocess
+import time
+
+import pytest
+
+log = logging.getLogger(__name__)
+
+
+@pytest.mark.skip_unless_on_linux
+def test_multimaster_dns(
+    salt_mm_master_1, salt_mm_minion_1, mm_master_1_salt_cli, etc_hosts, caplog
+):
+    """
+    Verify a minion configured with multimaster hot/hot will pick up a master's
+    dns change if it's been disconnected.
+    """
+
+    etc_hosts.write_text(
+        f"{etc_hosts.orig_text}\n172.16.0.1    master1.local master2.local"
+    )
+
+    log.info("Added hosts record for master1.local and master2.local")
+
+    with salt_mm_minion_1.started(start_timeout=180):
+        with caplog.at_level(logging.INFO):
+            ret = mm_master_1_salt_cli.run("test.ping", minion_tgt="mm-minion-1")
+            assert ret.returncode == 0
+            log.info("Removing secondary master IP address.")
+            etc_hosts.write_text(
+                f"{etc_hosts.orig_text}\n127.0.0.1    master1.local master2.local"
+            )
+            subprocess.check_output(["ip", "addr", "del", "172.16.0.1/32", "dev", "lo"])
+            log.info("Changed hosts record for master1.local and master2.local")
+            time.sleep(15)
+            assert (
+                "Master ip address changed from 172.16.0.1 to 127.0.0.1" in caplog.text
+            )
+            ret = mm_master_1_salt_cli.run("test.ping", minion_tgt="mm-minion-1")
+            assert ret.returncode == 0
+            assert (
+                "Master ip address changed from 172.16.0.1 to 127.0.0.1" in caplog.text
+            )

tests/pytests/scenarios/dns/test_dns.py

@@ -0,0 +1,29 @@
+import logging
+import subprocess
+import time
+
+import pytest
+
+log = logging.getLogger(__name__)
+
+
+@pytest.mark.skip_unless_on_linux
+def test_dns_change(master, minion, salt_cli, etc_hosts, caplog):
+    """
+    Verify a minion will pick up a master's dns change if it's been disconnected.
+    """
+
+    etc_hosts.write_text(f"{etc_hosts.orig_text}\n172.16.0.1    master.local")
+
+    with minion.started(start_timeout=180):
+        with caplog.at_level(logging.INFO):
+            ret = salt_cli.run("test.ping", minion_tgt="minion")
+            assert ret.returncode == 0
+            etc_hosts.write_text(f"{etc_hosts.orig_text}\n127.0.0.1    master.local")
+            subprocess.check_output(["ip", "addr", "del", "172.16.0.1/32", "dev", "lo"])
+            time.sleep(15)
+            assert (
+                "Master ip address changed from 172.16.0.1 to 127.0.0.1" in caplog.text
+            )
+            ret = salt_cli.run("test.ping", minion_tgt="minion")
+            assert ret.returncode == 0

tests/pytests/scenarios/failover/multimaster/test_failover_master.py

@@ -159,10 +159,6 @@ def test_minions_alive_with_no_master(
     """
     Make sure the minions stay alive after all masters have stopped.
     """
-    if grains["os_family"] == "Debian" and grains["osmajorrelease"] == 9:
-        pytest.skip(
-            "Skipping on Debian 9 until flaky issues resolved. See issue #61749"
-        )
     start_time = time.time()
     with salt_mm_failover_master_1.stopped():
         with salt_mm_failover_master_2.stopped():