Make cluster tests more stable

salt/master.py

@@ -2476,8 +2476,8 @@ class ClearFuncs(TransportMethods):
         # An alternative to copy may be to pop it
         # payload.pop("_stamp")
         self._send_ssh_pub(payload, ssh_minions=ssh_minions)
-        await self._send_pub(payload)
 
+        await self._send_pub(payload)
         return {
             "enc": "clear",
             "load": {"jid": clear_load["jid"], "minions": minions, "missing": missing},

salt/utils/event.py

@@ -706,8 +706,7 @@ class SaltEvent:
                 continue
             yield data
 
-    @tornado.gen.coroutine
-    def fire_event_async(self, data, tag, cb=None, timeout=1000):
+    async def fire_event_async(self, data, tag, cb=None, timeout=1000):
         """
         Send a single event into the publisher with payload dict "data" and
         event identifier "tag"

tests/pytests/integration/cluster/conftest.py

@@ -45,6 +45,12 @@ def cluster_master_1(request, salt_factories, cluster_pki_path, cluster_cache_pa
         ],
         "cluster_pki_dir": str(cluster_pki_path),
         "cache_dir": str(cluster_cache_path),
+        "log_granular_levels": {
+            "salt": "info",
+            "salt.transport": "debug",
+            "salt.channel": "debug",
+            "salt.utils.event": "debug",
+        },
     }
     factory = salt_factories.salt_master_daemon(
         "127.0.0.1",
@@ -74,6 +80,12 @@ def cluster_master_2(salt_factories, cluster_master_1):
         ],
         "cluster_pki_dir": cluster_master_1.config["cluster_pki_dir"],
         "cache_dir": cluster_master_1.config["cache_dir"],
+        "log_granular_levels": {
+            "salt": "info",
+            "salt.transport": "debug",
+            "salt.channel": "debug",
+            "salt.utils.event": "debug",
+        },
     }
 
     # Use the same ports for both masters, they are binding to different interfaces
@@ -110,6 +122,12 @@ def cluster_master_3(salt_factories, cluster_master_1):
         ],
         "cluster_pki_dir": cluster_master_1.config["cluster_pki_dir"],
         "cache_dir": cluster_master_1.config["cache_dir"],
+        "log_granular_levels": {
+            "salt": "info",
+            "salt.transport": "debug",
+            "salt.channel": "debug",
+            "salt.utils.event": "debug",
+        },
     }
 
     # Use the same ports for both masters, they are binding to different interfaces
@@ -139,6 +157,12 @@ def cluster_minion_1(cluster_master_1):
     config_overrides = {
         "master": f"{addr}:{port}",
         "test.foo": "baz",
+        "log_granular_levels": {
+            "salt": "info",
+            "salt.transport": "debug",
+            "salt.channel": "debug",
+            "salt.utils.event": "debug",
+        },
     }
     factory = cluster_master_1.salt_minion_daemon(
         "cluster-minion-1",

tests/pytests/integration/cluster/test_basic_cluster.py

@@ -1,3 +1,6 @@
+"""
+Cluster integration tests.
+"""
 import salt.utils.event
 
 

tests/pytests/scenarios/cluster/test_cluster.py

@@ -1,3 +1,6 @@
+"""
+Cluster scinarios.
+"""
 import os
 import pathlib
 import time
@@ -52,6 +55,8 @@ def test_cluster_key_rotation(
         if time.monotonic() - start > timeout:
             assert False, f"Drop file never removed {dfpath}"
 
+    time.sleep(30)
+
     keys = set()
 
     # Validate the aes session key for all masters match