Cluster aes session rotation test

tests/pytests/integration/cluster/conftest.py

@@ -0,0 +1,150 @@
+import logging
+import subprocess
+
+import pytest
+
+import salt.utils.platform
+
+log = logging.getLogger(__name__)
+
+
+@pytest.fixture
+def cluster_shared_path(tmp_path):
+    path = tmp_path / "cluster"
+    path.mkdir()
+    return path
+
+
+@pytest.fixture
+def cluster_pki_path(cluster_shared_path):
+    path = cluster_shared_path / "pki"
+    path.mkdir()
+    (path / "peers").mkdir()
+    return path
+
+
+@pytest.fixture
+def cluster_cache_path(cluster_shared_path):
+    path = cluster_shared_path / "cache"
+    path.mkdir()
+    return path
+
+
+@pytest.fixture
+def cluster_master_1(request, salt_factories, cluster_pki_path, cluster_cache_path):
+    config_defaults = {
+        "open_mode": True,
+        "transport": request.config.getoption("--transport"),
+    }
+    config_overrides = {
+        "interface": "127.0.0.1",
+        "cluster_id": "master_cluster",
+        "cluster_peers": [
+            "127.0.0.2",
+            "127.0.0.3",
+        ],
+        "cluster_pki_dir": str(cluster_pki_path),
+        "cache_dir": str(cluster_cache_path),
+    }
+    factory = salt_factories.salt_master_daemon(
+        "127.0.0.1",
+        defaults=config_defaults,
+        overrides=config_overrides,
+        extra_cli_arguments_after_first_start_failure=["--log-level=info"],
+    )
+    with factory.started(start_timeout=120):
+        yield factory
+
+
+@pytest.fixture
+def cluster_master_2(salt_factories, cluster_master_1):
+    if salt.utils.platform.is_darwin() or salt.utils.platform.is_freebsd():
+        subprocess.check_output(["ifconfig", "lo0", "alias", "127.0.0.2", "up"])
+
+    config_defaults = {
+        "open_mode": True,
+        "transport": cluster_master_1.config["transport"],
+    }
+    config_overrides = {
+        "interface": "127.0.0.2",
+        "cluster_id": "master_cluster",
+        "cluster_peers": [
+            "127.0.0.1",
+            "127.0.0.3",
+        ],
+        "cluster_pki_dir": cluster_master_1.config["cluster_pki_dir"],
+        "cache_dir": cluster_master_1.config["cache_dir"],
+    }
+
+    # Use the same ports for both masters, they are binding to different interfaces
+    for key in (
+        "ret_port",
+        "publish_port",
+    ):
+        config_overrides[key] = cluster_master_1.config[key]
+    factory = salt_factories.salt_master_daemon(
+        "127.0.0.2",
+        defaults=config_defaults,
+        overrides=config_overrides,
+        extra_cli_arguments_after_first_start_failure=["--log-level=info"],
+    )
+    with factory.started(start_timeout=120):
+        yield factory
+
+
+@pytest.fixture
+def cluster_master_3(salt_factories, cluster_master_1):
+    if salt.utils.platform.is_darwin() or salt.utils.platform.is_freebsd():
+        subprocess.check_output(["ifconfig", "lo0", "alias", "127.0.0.3", "up"])
+
+    config_defaults = {
+        "open_mode": True,
+        "transport": cluster_master_1.config["transport"],
+    }
+    config_overrides = {
+        "interface": "127.0.0.3",
+        "cluster_id": "master_cluster",
+        "cluster_peers": [
+            "127.0.0.1",
+            "127.0.0.2",
+        ],
+        "cluster_pki_dir": cluster_master_1.config["cluster_pki_dir"],
+        "cache_dir": cluster_master_1.config["cache_dir"],
+    }
+
+    # Use the same ports for both masters, they are binding to different interfaces
+    for key in (
+        "ret_port",
+        "publish_port",
+    ):
+        config_overrides[key] = cluster_master_1.config[key]
+    factory = salt_factories.salt_master_daemon(
+        "127.0.0.3",
+        defaults=config_defaults,
+        overrides=config_overrides,
+        extra_cli_arguments_after_first_start_failure=["--log-level=info"],
+    )
+    with factory.started(start_timeout=120):
+        yield factory
+
+
+@pytest.fixture
+def cluster_minion_1(cluster_master_1):
+    config_defaults = {
+        "transport": cluster_master_1.config["transport"],
+    }
+
+    port = cluster_master_1.config["ret_port"]
+    addr = cluster_master_1.config["interface"]
+    config_overrides = {
+        "master": f"{addr}:{port}",
+        "test.foo": "baz",
+    }
+    factory = cluster_master_1.salt_minion_daemon(
+        "cluster-minion-1",
+        defaults=config_defaults,
+        overrides=config_overrides,
+        extra_cli_arguments_after_first_start_failure=["--log-level=info"],
+    )
+    with factory.started(start_timeout=120):
+        yield factory

tests/pytests/scenarios/cluster/conftest.py

@@ -5,146 +5,16 @@ import pytest
 
 import salt.utils.platform
 
+
+from tests.pytests.integration.cluster.conftest import (
+    cluster_shared_path,
+    cluster_pki_path,
+    cluster_cache_path,
+    cluster_master_1,
+    cluster_master_2,
+    cluster_master_3,
+    cluster_minion_1,
+)
+
 log = logging.getLogger(__name__)
 
-
-@pytest.fixture
-def cluster_shared_path(tmp_path):
-    path = tmp_path / "cluster"
-    path.mkdir()
-    return path
-
-
-@pytest.fixture
-def cluster_pki_path(cluster_shared_path):
-    path = cluster_shared_path / "pki"
-    path.mkdir()
-    (path / "peers").mkdir()
-    return path
-
-
-@pytest.fixture
-def cluster_cache_path(cluster_shared_path):
-    path = cluster_shared_path / "cache"
-    path.mkdir()
-    return path
-
-
-@pytest.fixture
-def cluster_master_1(request, salt_factories, cluster_pki_path, cluster_cache_path):
-    config_defaults = {
-        "open_mode": True,
-        "transport": request.config.getoption("--transport"),
-    }
-    config_overrides = {
-        "interface": "127.0.0.1",
-        "cluster_id": "master_cluster",
-        "cluster_peers": [
-            "127.0.0.2",
-            "127.0.0.3",
-        ],
-        "cluster_pki_dir": str(cluster_pki_path),
-        "cache_dir": str(cluster_cache_path),
-    }
-    factory = salt_factories.salt_master_daemon(
-        "127.0.0.1",
-        defaults=config_defaults,
-        overrides=config_overrides,
-        extra_cli_arguments_after_first_start_failure=["--log-level=info"],
-    )
-    with factory.started(start_timeout=120):
-        yield factory
-
-
-@pytest.fixture
-def cluster_master_2(salt_factories, cluster_master_1):
-    if salt.utils.platform.is_darwin() or salt.utils.platform.is_freebsd():
-        subprocess.check_output(["ifconfig", "lo0", "alias", "127.0.0.2", "up"])
-
-    config_defaults = {
-        "open_mode": True,
-        "transport": cluster_master_1.config["transport"],
-    }
-    config_overrides = {
-        "interface": "127.0.0.2",
-        "cluster_id": "master_cluster",
-        "cluster_peers": [
-            "127.0.0.1",
-            "127.0.0.3",
-        ],
-        "cluster_pki_dir": cluster_master_1.config["cluster_pki_dir"],
-        "cache_dir": cluster_master_1.config["cache_dir"],
-    }
-
-    # Use the same ports for both masters, they are binding to different interfaces
-    for key in (
-        "ret_port",
-        "publish_port",
-    ):
-        config_overrides[key] = cluster_master_1.config[key]
-    factory = salt_factories.salt_master_daemon(
-        "127.0.0.2",
-        defaults=config_defaults,
-        overrides=config_overrides,
-        extra_cli_arguments_after_first_start_failure=["--log-level=info"],
-    )
-    with factory.started(start_timeout=120):
-        yield factory
-
-
-@pytest.fixture
-def cluster_master_3(salt_factories, cluster_master_1):
-    if salt.utils.platform.is_darwin() or salt.utils.platform.is_freebsd():
-        subprocess.check_output(["ifconfig", "lo0", "alias", "127.0.0.3", "up"])
-
-    config_defaults = {
-        "open_mode": True,
-        "transport": cluster_master_1.config["transport"],
-    }
-    config_overrides = {
-        "interface": "127.0.0.3",
-        "cluster_id": "master_cluster",
-        "cluster_peers": [
-            "127.0.0.1",
-            "127.0.0.2",
-        ],
-        "cluster_pki_dir": cluster_master_1.config["cluster_pki_dir"],
-        "cache_dir": cluster_master_1.config["cache_dir"],
-    }
-
-    # Use the same ports for both masters, they are binding to different interfaces
-    for key in (
-        "ret_port",
-        "publish_port",
-    ):
-        config_overrides[key] = cluster_master_1.config[key]
-    factory = salt_factories.salt_master_daemon(
-        "127.0.0.3",
-        defaults=config_defaults,
-        overrides=config_overrides,
-        extra_cli_arguments_after_first_start_failure=["--log-level=info"],
-    )
-    with factory.started(start_timeout=120):
-        yield factory
-
-
-@pytest.fixture
-def cluster_minion_1(cluster_master_1):
-    config_defaults = {
-        "transport": cluster_master_1.config["transport"],
-    }
-
-    port = cluster_master_1.config["ret_port"]
-    addr = cluster_master_1.config["interface"]
-    config_overrides = {
-        "master": f"{addr}:{port}",
-        "test.foo": "baz",
-    }
-    factory = cluster_master_1.salt_minion_daemon(
-        "cluster-minion-1",
-        defaults=config_defaults,
-        overrides=config_overrides,
-        extra_cli_arguments_after_first_start_failure=["--log-level=info"],
-    )
-    with factory.started(start_timeout=120):
-        yield factory

tests/pytests/scenarios/cluster/test_cluster.py

@@ -0,0 +1,57 @@
+import pathlib
+import time
+import os
+
+import salt.crypt
+
+def test_cluster_key_rotation(
+    cluster_master_1, cluster_master_2, cluster_master_3, cluster_minion_1,
+    cluster_cache_path,
+):
+    cli = cluster_master_2.salt_cli(timeout=120)
+    ret = cli.run("test.ping", minion_tgt="cluster-minion-1")
+    assert ret.data is True
+
+    # Validate the aes session key for all masters match
+    keys = set()
+    for master in (cluster_master_1, cluster_master_2, cluster_master_3,):
+        config = cluster_minion_1.config.copy()
+        config["master_uri"] = f"tcp://{master.config['interface']}:{master.config['ret_port']}"
+        auth = salt.crypt.SAuth(config)
+        auth.authenticate()
+        assert "aes" in auth._creds
+        keys.add(auth._creds["aes"])
+
+    assert len(keys) == 1
+    orig_aes = keys.pop()
+
+    dfpath = pathlib.Path(cluster_master_1.config["cachedir"]) / ".dfn"
+    assert not dfpath.exists()
+    salt.crypt.dropfile(
+        cluster_master_1.config["cachedir"],
+        user=os.getlogin(),
+        master_id=cluster_master_1.config["id"],
+    )
+    assert dfpath.exists()
+    timeout = 2 * cluster_master_1.config["loop_interval"]
+    start = time.monotonic()
+    while True:
+        if not dfpath.exists():
+            break
+        if time.monotonic() - start > timeout:
+            assert False, f"Drop file never removed {dfpath}"
+
+    keys = set()
+
+    # Validate the aes session key for all masters match
+    for master in (cluster_master_1, cluster_master_2, cluster_master_3,):
+        config = cluster_minion_1.config.copy()
+        config["master_uri"] = f"tcp://{master.config['interface']}:{master.config['ret_port']}"
+        auth = salt.crypt.SAuth(config)
+        auth.authenticate()
+        assert "aes" in auth._creds
+        keys.add(auth._creds["aes"])
+
+    assert len(keys) == 1
+    # Validate the aes session key actually changed
+    assert orig_aes != keys.pop()