mirror of
https://github.com/saltstack/salt.git
synced 2025-04-17 10:10:20 +00:00
807 lines
28 KiB
Python
807 lines
28 KiB
Python
"""
|
|
:codeauthor: Thomas Jackson <jacksontj.89@gmail.com>
|
|
"""
|
|
|
|
import ctypes
|
|
import multiprocessing
|
|
import os
|
|
import threading
|
|
import time
|
|
from concurrent.futures.thread import ThreadPoolExecutor
|
|
|
|
import salt.config
|
|
import salt.exceptions
|
|
import salt.ext.tornado.gen
|
|
import salt.ext.tornado.ioloop
|
|
import salt.log.setup
|
|
import salt.transport.client
|
|
import salt.transport.server
|
|
import salt.utils.platform
|
|
import salt.utils.process
|
|
import zmq.eventloop.ioloop
|
|
from salt.ext import six
|
|
from salt.ext.six.moves import range
|
|
from salt.ext.tornado.testing import AsyncTestCase
|
|
from salt.transport.zeromq import AsyncReqMessageClientPool
|
|
from saltfactories.utils.ports import get_unused_localhost_port
|
|
from tests.support.helpers import flaky, not_runs_on, slowTest
|
|
from tests.support.mixins import AdaptedConfigurationTestCaseMixin
|
|
from tests.support.mock import MagicMock, call, patch
|
|
from tests.support.runtests import RUNTIME_VARS
|
|
from tests.support.unit import TestCase, skipIf
|
|
from tests.unit.transport.mixins import (
|
|
PubChannelMixin,
|
|
ReqChannelMixin,
|
|
run_loop_in_thread,
|
|
)
|
|
|
|
x = "fix pre"
|
|
|
|
# support pyzmq 13.0.x, TODO: remove once we force people to 14.0.x
|
|
if not hasattr(zmq.eventloop.ioloop, "ZMQIOLoop"):
|
|
zmq.eventloop.ioloop.ZMQIOLoop = zmq.eventloop.ioloop.IOLoop
|
|
|
|
|
|
class BaseZMQReqCase(TestCase, AdaptedConfigurationTestCaseMixin):
|
|
"""
|
|
Test the req server/client pair
|
|
"""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
if not hasattr(cls, "_handle_payload"):
|
|
return
|
|
ret_port = get_unused_localhost_port()
|
|
publish_port = get_unused_localhost_port()
|
|
tcp_master_pub_port = get_unused_localhost_port()
|
|
tcp_master_pull_port = get_unused_localhost_port()
|
|
tcp_master_publish_pull = get_unused_localhost_port()
|
|
tcp_master_workers = get_unused_localhost_port()
|
|
cls.master_config = cls.get_temp_config(
|
|
"master",
|
|
**{
|
|
"transport": "zeromq",
|
|
"auto_accept": True,
|
|
"ret_port": ret_port,
|
|
"publish_port": publish_port,
|
|
"tcp_master_pub_port": tcp_master_pub_port,
|
|
"tcp_master_pull_port": tcp_master_pull_port,
|
|
"tcp_master_publish_pull": tcp_master_publish_pull,
|
|
"tcp_master_workers": tcp_master_workers,
|
|
}
|
|
)
|
|
|
|
cls.minion_config = cls.get_temp_config(
|
|
"minion",
|
|
**{
|
|
"transport": "zeromq",
|
|
"master_ip": "127.0.0.1",
|
|
"master_port": ret_port,
|
|
"auth_timeout": 5,
|
|
"auth_tries": 1,
|
|
"master_uri": "tcp://127.0.0.1:{}".format(ret_port),
|
|
}
|
|
)
|
|
|
|
cls.process_manager = salt.utils.process.ProcessManager(
|
|
name="ReqServer_ProcessManager"
|
|
)
|
|
|
|
cls.server_channel = salt.transport.server.ReqServerChannel.factory(
|
|
cls.master_config
|
|
)
|
|
cls.server_channel.pre_fork(cls.process_manager)
|
|
|
|
cls.io_loop = salt.ext.tornado.ioloop.IOLoop()
|
|
cls.evt = threading.Event()
|
|
cls.server_channel.post_fork(cls._handle_payload, io_loop=cls.io_loop)
|
|
cls.server_thread = threading.Thread(
|
|
target=run_loop_in_thread, args=(cls.io_loop, cls.evt)
|
|
)
|
|
cls.server_thread.start()
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
if not hasattr(cls, "_handle_payload"):
|
|
return
|
|
# Attempting to kill the children hangs the test suite.
|
|
# Let the test suite handle this instead.
|
|
cls.process_manager.stop_restarting()
|
|
cls.process_manager.kill_children()
|
|
cls.evt.set()
|
|
cls.server_thread.join()
|
|
time.sleep(
|
|
2
|
|
) # Give the procs a chance to fully close before we stop the io_loop
|
|
cls.server_channel.close()
|
|
del cls.server_channel
|
|
del cls.io_loop
|
|
del cls.process_manager
|
|
del cls.server_thread
|
|
del cls.master_config
|
|
del cls.minion_config
|
|
|
|
@classmethod
|
|
def _handle_payload(cls, payload):
|
|
"""
|
|
TODO: something besides echo
|
|
"""
|
|
return payload, {"fun": "send_clear"}
|
|
|
|
|
|
class ClearReqTestCases(BaseZMQReqCase, ReqChannelMixin):
|
|
"""
|
|
Test all of the clear msg stuff
|
|
"""
|
|
|
|
def setUp(self):
|
|
self.channel = salt.transport.client.ReqChannel.factory(
|
|
self.minion_config, crypt="clear"
|
|
)
|
|
|
|
def tearDown(self):
|
|
self.channel.close()
|
|
del self.channel
|
|
|
|
@classmethod
|
|
@salt.ext.tornado.gen.coroutine
|
|
def _handle_payload(cls, payload):
|
|
"""
|
|
TODO: something besides echo
|
|
"""
|
|
raise salt.ext.tornado.gen.Return((payload, {"fun": "send_clear"}))
|
|
|
|
@slowTest
|
|
def test_master_uri_override(self):
|
|
"""
|
|
ensure master_uri kwarg is respected
|
|
"""
|
|
# minion_config should be 127.0.0.1, we want a different uri that still connects
|
|
uri = "tcp://{master_ip}:{master_port}".format(
|
|
master_ip="localhost", master_port=self.minion_config["master_port"]
|
|
)
|
|
|
|
channel = salt.transport.client.ReqChannel.factory(
|
|
self.minion_config, master_uri=uri
|
|
)
|
|
self.assertIn("localhost", channel.master_uri)
|
|
del channel
|
|
|
|
|
|
@flaky
|
|
@not_runs_on(
|
|
kernel="linux",
|
|
os_familiy="Suse",
|
|
reason="Skipping until https://github.com/saltstack/salt/issues/32902 gets fixed",
|
|
)
|
|
class AESReqTestCases(BaseZMQReqCase, ReqChannelMixin):
|
|
def setUp(self):
|
|
self.channel = salt.transport.client.ReqChannel.factory(self.minion_config)
|
|
|
|
def tearDown(self):
|
|
self.channel.close()
|
|
del self.channel
|
|
|
|
@classmethod
|
|
@salt.ext.tornado.gen.coroutine
|
|
def _handle_payload(cls, payload):
|
|
"""
|
|
TODO: something besides echo
|
|
"""
|
|
raise salt.ext.tornado.gen.Return((payload, {"fun": "send"}))
|
|
|
|
# TODO: make failed returns have a specific framing so we can raise the same exception
|
|
# on encrypted channels
|
|
#
|
|
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
#
|
|
# WARNING: This test will fail randomly on any system with > 1 CPU core!!!
|
|
#
|
|
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
@slowTest
|
|
def test_badload(self):
|
|
"""
|
|
Test a variety of bad requests, make sure that we get some sort of error
|
|
"""
|
|
# TODO: This test should be re-enabled when Jenkins moves to C7.
|
|
# Once the version of salt-testing is increased to something newer than the September
|
|
# release of salt-testing, the @flaky decorator should be applied to this test.
|
|
msgs = ["", [], tuple()]
|
|
for msg in msgs:
|
|
with self.assertRaises(salt.exceptions.AuthenticationError):
|
|
ret = self.channel.send(msg, timeout=5)
|
|
|
|
|
|
class BaseZMQPubCase(AsyncTestCase, AdaptedConfigurationTestCaseMixin):
|
|
"""
|
|
Test the req server/client pair
|
|
"""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
ret_port = get_unused_localhost_port()
|
|
publish_port = get_unused_localhost_port()
|
|
tcp_master_pub_port = get_unused_localhost_port()
|
|
tcp_master_pull_port = get_unused_localhost_port()
|
|
tcp_master_publish_pull = get_unused_localhost_port()
|
|
tcp_master_workers = get_unused_localhost_port()
|
|
cls.master_config = cls.get_temp_config(
|
|
"master",
|
|
**{
|
|
"transport": "zeromq",
|
|
"auto_accept": True,
|
|
"ret_port": ret_port,
|
|
"publish_port": publish_port,
|
|
"tcp_master_pub_port": tcp_master_pub_port,
|
|
"tcp_master_pull_port": tcp_master_pull_port,
|
|
"tcp_master_publish_pull": tcp_master_publish_pull,
|
|
"tcp_master_workers": tcp_master_workers,
|
|
}
|
|
)
|
|
|
|
cls.minion_config = salt.config.minion_config(
|
|
os.path.join(RUNTIME_VARS.TMP_CONF_DIR, "minion")
|
|
)
|
|
cls.minion_config = cls.get_temp_config(
|
|
"minion",
|
|
**{
|
|
"transport": "zeromq",
|
|
"master_ip": "127.0.0.1",
|
|
"master_port": ret_port,
|
|
"master_uri": "tcp://127.0.0.1:{}".format(ret_port),
|
|
}
|
|
)
|
|
|
|
cls.process_manager = salt.utils.process.ProcessManager(
|
|
name="ReqServer_ProcessManager"
|
|
)
|
|
|
|
cls.server_channel = salt.transport.server.PubServerChannel.factory(
|
|
cls.master_config
|
|
)
|
|
cls.server_channel.pre_fork(cls.process_manager)
|
|
|
|
# we also require req server for auth
|
|
cls.req_server_channel = salt.transport.server.ReqServerChannel.factory(
|
|
cls.master_config
|
|
)
|
|
cls.req_server_channel.pre_fork(cls.process_manager)
|
|
|
|
cls._server_io_loop = salt.ext.tornado.ioloop.IOLoop()
|
|
cls.evt = threading.Event()
|
|
cls.req_server_channel.post_fork(
|
|
cls._handle_payload, io_loop=cls._server_io_loop
|
|
)
|
|
cls.server_thread = threading.Thread(
|
|
target=run_loop_in_thread, args=(cls._server_io_loop, cls.evt)
|
|
)
|
|
cls.server_thread.start()
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
cls.process_manager.kill_children()
|
|
cls.process_manager.stop_restarting()
|
|
time.sleep(
|
|
2
|
|
) # Give the procs a chance to fully close before we stop the io_loop
|
|
cls.evt.set()
|
|
cls.server_thread.join()
|
|
cls.req_server_channel.close()
|
|
cls.server_channel.close()
|
|
cls._server_io_loop.stop()
|
|
del cls.server_channel
|
|
del cls._server_io_loop
|
|
del cls.process_manager
|
|
del cls.server_thread
|
|
del cls.master_config
|
|
del cls.minion_config
|
|
|
|
@classmethod
|
|
def _handle_payload(cls, payload):
|
|
"""
|
|
TODO: something besides echo
|
|
"""
|
|
return payload, {"fun": "send_clear"}
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
self._start_handlers = dict(self.io_loop._handlers)
|
|
|
|
def tearDown(self):
|
|
super().tearDown()
|
|
failures = []
|
|
for k, v in self.io_loop._handlers.items():
|
|
if self._start_handlers.get(k) != v:
|
|
failures.append((k, v))
|
|
del self._start_handlers
|
|
if len(failures) > 0:
|
|
raise Exception("FDs still attached to the IOLoop: {}".format(failures))
|
|
|
|
|
|
@skipIf(True, "Skip until we can devote time to fix this test")
|
|
class AsyncPubChannelTest(BaseZMQPubCase, PubChannelMixin):
|
|
"""
|
|
Tests around the publish system
|
|
"""
|
|
|
|
def get_new_ioloop(self):
|
|
return salt.ext.tornado.ioloop.IOLoop()
|
|
|
|
|
|
class AsyncReqMessageClientPoolTest(TestCase):
|
|
def setUp(self):
|
|
super().setUp()
|
|
sock_pool_size = 5
|
|
with patch(
|
|
"salt.transport.zeromq.AsyncReqMessageClient.__init__",
|
|
MagicMock(return_value=None),
|
|
):
|
|
self.message_client_pool = AsyncReqMessageClientPool(
|
|
{"sock_pool_size": sock_pool_size}, args=({}, "")
|
|
)
|
|
self.original_message_clients = self.message_client_pool.message_clients
|
|
self.message_client_pool.message_clients = [
|
|
MagicMock() for _ in range(sock_pool_size)
|
|
]
|
|
|
|
def tearDown(self):
|
|
del self.original_message_clients
|
|
super().tearDown()
|
|
|
|
def test_send(self):
|
|
for message_client_mock in self.message_client_pool.message_clients:
|
|
message_client_mock.send_queue = [0, 0, 0]
|
|
message_client_mock.send.return_value = []
|
|
|
|
self.assertEqual([], self.message_client_pool.send())
|
|
|
|
self.message_client_pool.message_clients[2].send_queue = [0]
|
|
self.message_client_pool.message_clients[2].send.return_value = [1]
|
|
self.assertEqual([1], self.message_client_pool.send())
|
|
|
|
|
|
class ZMQConfigTest(TestCase):
|
|
def test_master_uri(self):
|
|
"""
|
|
test _get_master_uri method
|
|
"""
|
|
|
|
m_ip = "127.0.0.1"
|
|
m_port = 4505
|
|
s_ip = "111.1.0.1"
|
|
s_port = 4058
|
|
|
|
m_ip6 = "1234:5678::9abc"
|
|
s_ip6 = "1234:5678::1:9abc"
|
|
|
|
with patch("salt.transport.zeromq.LIBZMQ_VERSION_INFO", (4, 1, 6)), patch(
|
|
"salt.transport.zeromq.ZMQ_VERSION_INFO", (16, 0, 1)
|
|
):
|
|
# pass in both source_ip and source_port
|
|
assert salt.transport.zeromq._get_master_uri(
|
|
master_ip=m_ip, master_port=m_port, source_ip=s_ip, source_port=s_port
|
|
) == "tcp://{}:{};{}:{}".format(s_ip, s_port, m_ip, m_port)
|
|
|
|
assert salt.transport.zeromq._get_master_uri(
|
|
master_ip=m_ip6, master_port=m_port, source_ip=s_ip6, source_port=s_port
|
|
) == "tcp://[{}]:{};[{}]:{}".format(s_ip6, s_port, m_ip6, m_port)
|
|
|
|
# source ip and source_port empty
|
|
assert salt.transport.zeromq._get_master_uri(
|
|
master_ip=m_ip, master_port=m_port
|
|
) == "tcp://{}:{}".format(m_ip, m_port)
|
|
|
|
assert salt.transport.zeromq._get_master_uri(
|
|
master_ip=m_ip6, master_port=m_port
|
|
) == "tcp://[{}]:{}".format(m_ip6, m_port)
|
|
|
|
# pass in only source_ip
|
|
assert salt.transport.zeromq._get_master_uri(
|
|
master_ip=m_ip, master_port=m_port, source_ip=s_ip
|
|
) == "tcp://{}:0;{}:{}".format(s_ip, m_ip, m_port)
|
|
|
|
assert salt.transport.zeromq._get_master_uri(
|
|
master_ip=m_ip6, master_port=m_port, source_ip=s_ip6
|
|
) == "tcp://[{}]:0;[{}]:{}".format(s_ip6, m_ip6, m_port)
|
|
|
|
# pass in only source_port
|
|
assert salt.transport.zeromq._get_master_uri(
|
|
master_ip=m_ip, master_port=m_port, source_port=s_port
|
|
) == "tcp://0.0.0.0:{};{}:{}".format(s_port, m_ip, m_port)
|
|
|
|
|
|
class PubServerChannel(TestCase, AdaptedConfigurationTestCaseMixin):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
ret_port = get_unused_localhost_port()
|
|
publish_port = get_unused_localhost_port()
|
|
tcp_master_pub_port = get_unused_localhost_port()
|
|
tcp_master_pull_port = get_unused_localhost_port()
|
|
tcp_master_publish_pull = get_unused_localhost_port()
|
|
tcp_master_workers = get_unused_localhost_port()
|
|
cls.master_config = cls.get_temp_config(
|
|
"master",
|
|
**{
|
|
"transport": "zeromq",
|
|
"auto_accept": True,
|
|
"ret_port": ret_port,
|
|
"publish_port": publish_port,
|
|
"tcp_master_pub_port": tcp_master_pub_port,
|
|
"tcp_master_pull_port": tcp_master_pull_port,
|
|
"tcp_master_publish_pull": tcp_master_publish_pull,
|
|
"tcp_master_workers": tcp_master_workers,
|
|
"sign_pub_messages": False,
|
|
}
|
|
)
|
|
salt.master.SMaster.secrets["aes"] = {
|
|
"secret": multiprocessing.Array(
|
|
ctypes.c_char, six.b(salt.crypt.Crypticle.generate_key_string()),
|
|
),
|
|
}
|
|
cls.minion_config = cls.get_temp_config(
|
|
"minion",
|
|
**{
|
|
"transport": "zeromq",
|
|
"master_ip": "127.0.0.1",
|
|
"master_port": ret_port,
|
|
"auth_timeout": 5,
|
|
"auth_tries": 1,
|
|
"master_uri": "tcp://127.0.0.1:{}".format(ret_port),
|
|
}
|
|
)
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
del cls.minion_config
|
|
del cls.master_config
|
|
|
|
def setUp(self):
|
|
# Start the event loop, even though we don't directly use this with
|
|
# ZeroMQPubServerChannel, having it running seems to increase the
|
|
# likely hood of dropped messages.
|
|
self.io_loop = salt.ext.tornado.ioloop.IOLoop()
|
|
self.io_loop.make_current()
|
|
self.io_loop_thread = threading.Thread(target=self.io_loop.start)
|
|
self.io_loop_thread.start()
|
|
self.process_manager = salt.utils.process.ProcessManager(
|
|
name="PubServer_ProcessManager"
|
|
)
|
|
|
|
def tearDown(self):
|
|
self.io_loop.add_callback(self.io_loop.stop)
|
|
self.io_loop_thread.join()
|
|
self.process_manager.stop_restarting()
|
|
self.process_manager.kill_children()
|
|
del self.io_loop
|
|
del self.io_loop_thread
|
|
del self.process_manager
|
|
|
|
@staticmethod
|
|
def _gather_results(opts, pub_uri, results, timeout=120, messages=None):
|
|
"""
|
|
Gather results until then number of seconds specified by timeout passes
|
|
without reveiving a message
|
|
"""
|
|
ctx = zmq.Context()
|
|
sock = ctx.socket(zmq.SUB)
|
|
sock.setsockopt(zmq.LINGER, -1)
|
|
sock.setsockopt(zmq.SUBSCRIBE, b"")
|
|
sock.connect(pub_uri)
|
|
last_msg = time.time()
|
|
serial = salt.payload.Serial(opts)
|
|
crypticle = salt.crypt.Crypticle(
|
|
opts, salt.master.SMaster.secrets["aes"]["secret"].value
|
|
)
|
|
while time.time() - last_msg < timeout:
|
|
try:
|
|
payload = sock.recv(zmq.NOBLOCK)
|
|
except zmq.ZMQError:
|
|
time.sleep(0.01)
|
|
else:
|
|
if messages:
|
|
if messages != 1:
|
|
messages -= 1
|
|
continue
|
|
payload = crypticle.loads(serial.loads(payload)["load"])
|
|
if "stop" in payload:
|
|
break
|
|
last_msg = time.time()
|
|
results.append(payload["jid"])
|
|
|
|
@skipIf(salt.utils.platform.is_windows(), "Skip on Windows OS")
|
|
@slowTest
|
|
def test_publish_to_pubserv_ipc(self):
|
|
"""
|
|
Test sending 10K messags to ZeroMQPubServerChannel using IPC transport
|
|
|
|
ZMQ's ipc transport not supported on Windows
|
|
"""
|
|
opts = dict(self.master_config, ipc_mode="ipc", pub_hwm=0)
|
|
server_channel = salt.transport.zeromq.ZeroMQPubServerChannel(opts)
|
|
server_channel.pre_fork(
|
|
self.process_manager,
|
|
kwargs={"log_queue": salt.log.setup.get_multiprocessing_logging_queue()},
|
|
)
|
|
pub_uri = "tcp://{interface}:{publish_port}".format(**server_channel.opts)
|
|
send_num = 10000
|
|
expect = []
|
|
results = []
|
|
gather = threading.Thread(
|
|
target=self._gather_results, args=(self.minion_config, pub_uri, results,)
|
|
)
|
|
gather.start()
|
|
# Allow time for server channel to start, especially on windows
|
|
time.sleep(2)
|
|
for i in range(send_num):
|
|
expect.append(i)
|
|
load = {"tgt_type": "glob", "tgt": "*", "jid": i}
|
|
server_channel.publish(load)
|
|
server_channel.publish({"tgt_type": "glob", "tgt": "*", "stop": True})
|
|
gather.join()
|
|
server_channel.pub_close()
|
|
assert len(results) == send_num, (len(results), set(expect).difference(results))
|
|
|
|
@skipIf(salt.utils.platform.is_linux(), "Skip on Linux")
|
|
@slowTest
|
|
def test_zeromq_publish_port(self):
|
|
"""
|
|
test when connecting that we
|
|
use the publish_port set in opts
|
|
when its not 4506
|
|
"""
|
|
opts = dict(
|
|
self.master_config,
|
|
ipc_mode="ipc",
|
|
pub_hwm=0,
|
|
recon_randomize=False,
|
|
publish_port=455505,
|
|
recon_default=1,
|
|
recon_max=2,
|
|
master_ip="127.0.0.1",
|
|
acceptance_wait_time=5,
|
|
acceptance_wait_time_max=5,
|
|
)
|
|
opts["master_uri"] = "tcp://{interface}:{publish_port}".format(**opts)
|
|
|
|
channel = salt.transport.zeromq.AsyncZeroMQPubChannel(opts)
|
|
patch_socket = MagicMock(return_value=True)
|
|
patch_auth = MagicMock(return_value=True)
|
|
with patch.object(channel, "_socket", patch_socket), patch.object(
|
|
channel, "auth", patch_auth
|
|
):
|
|
channel.connect()
|
|
assert str(opts["publish_port"]) in patch_socket.mock_calls[0][1][0]
|
|
|
|
@skipIf(salt.utils.platform.is_linux(), "Skip on Linux")
|
|
def test_zeromq_zeromq_filtering_decode_message_no_match(self):
|
|
"""
|
|
test AsyncZeroMQPubChannel _decode_messages when
|
|
zmq_filtering enabled and minion does not match
|
|
"""
|
|
message = [
|
|
b"4f26aeafdb2367620a393c973eddbe8f8b846eb",
|
|
b"\x82\xa3enc\xa3aes\xa4load\xda\x00`\xeeR\xcf"
|
|
b"\x0eaI#V\x17if\xcf\xae\x05\xa7\xb3bN\xf7\xb2\xe2"
|
|
b'\xd0sF\xd1\xd4\xecB\xe8\xaf"/*ml\x80Q3\xdb\xaexg'
|
|
b"\x8e\x8a\x8c\xd3l\x03\\,J\xa7\x01i\xd1:]\xe3\x8d"
|
|
b"\xf4\x03\x88K\x84\n`\xe8\x9a\xad\xad\xc6\x8ea\x15>"
|
|
b"\x92m\x9e\xc7aM\x11?\x18;\xbd\x04c\x07\x85\x99\xa3\xea[\x00D",
|
|
]
|
|
|
|
opts = dict(
|
|
self.master_config,
|
|
ipc_mode="ipc",
|
|
pub_hwm=0,
|
|
zmq_filtering=True,
|
|
recon_randomize=False,
|
|
recon_default=1,
|
|
recon_max=2,
|
|
master_ip="127.0.0.1",
|
|
acceptance_wait_time=5,
|
|
acceptance_wait_time_max=5,
|
|
)
|
|
opts["master_uri"] = "tcp://{interface}:{publish_port}".format(**opts)
|
|
|
|
server_channel = salt.transport.zeromq.AsyncZeroMQPubChannel(opts)
|
|
with patch(
|
|
"salt.crypt.AsyncAuth.crypticle",
|
|
MagicMock(return_value={"tgt_type": "glob", "tgt": "*", "jid": 1}),
|
|
) as mock_test:
|
|
res = server_channel._decode_messages(message)
|
|
assert res.result() is None
|
|
|
|
@skipIf(salt.utils.platform.is_linux(), "Skip on Linux")
|
|
def test_zeromq_zeromq_filtering_decode_message(self):
|
|
"""
|
|
test AsyncZeroMQPubChannel _decode_messages
|
|
when zmq_filtered enabled
|
|
"""
|
|
message = [
|
|
b"4f26aeafdb2367620a393c973eddbe8f8b846ebd",
|
|
b"\x82\xa3enc\xa3aes\xa4load\xda\x00`\xeeR\xcf"
|
|
b"\x0eaI#V\x17if\xcf\xae\x05\xa7\xb3bN\xf7\xb2\xe2"
|
|
b'\xd0sF\xd1\xd4\xecB\xe8\xaf"/*ml\x80Q3\xdb\xaexg'
|
|
b"\x8e\x8a\x8c\xd3l\x03\\,J\xa7\x01i\xd1:]\xe3\x8d"
|
|
b"\xf4\x03\x88K\x84\n`\xe8\x9a\xad\xad\xc6\x8ea\x15>"
|
|
b"\x92m\x9e\xc7aM\x11?\x18;\xbd\x04c\x07\x85\x99\xa3\xea[\x00D",
|
|
]
|
|
|
|
opts = dict(
|
|
self.master_config,
|
|
ipc_mode="ipc",
|
|
pub_hwm=0,
|
|
zmq_filtering=True,
|
|
recon_randomize=False,
|
|
recon_default=1,
|
|
recon_max=2,
|
|
master_ip="127.0.0.1",
|
|
acceptance_wait_time=5,
|
|
acceptance_wait_time_max=5,
|
|
)
|
|
opts["master_uri"] = "tcp://{interface}:{publish_port}".format(**opts)
|
|
|
|
server_channel = salt.transport.zeromq.AsyncZeroMQPubChannel(opts)
|
|
with patch(
|
|
"salt.crypt.AsyncAuth.crypticle",
|
|
MagicMock(return_value={"tgt_type": "glob", "tgt": "*", "jid": 1}),
|
|
) as mock_test:
|
|
res = server_channel._decode_messages(message)
|
|
|
|
assert res.result()["enc"] == "aes"
|
|
|
|
@skipIf(salt.utils.platform.is_windows(), "Skip on Windows OS")
|
|
@slowTest
|
|
def test_zeromq_filtering(self):
|
|
"""
|
|
Test sending messags to publisher using UDP
|
|
with zeromq_filtering enabled
|
|
"""
|
|
opts = dict(
|
|
self.master_config,
|
|
ipc_mode="ipc",
|
|
pub_hwm=0,
|
|
zmq_filtering=True,
|
|
acceptance_wait_time=5,
|
|
)
|
|
server_channel = salt.transport.zeromq.ZeroMQPubServerChannel(opts)
|
|
server_channel.pre_fork(
|
|
self.process_manager,
|
|
kwargs={"log_queue": salt.log.setup.get_multiprocessing_logging_queue()},
|
|
)
|
|
pub_uri = "tcp://{interface}:{publish_port}".format(**server_channel.opts)
|
|
send_num = 1
|
|
expect = []
|
|
results = []
|
|
gather = threading.Thread(
|
|
target=self._gather_results,
|
|
args=(self.minion_config, pub_uri, results,),
|
|
kwargs={"messages": 2},
|
|
)
|
|
gather.start()
|
|
# Allow time for server channel to start, especially on windows
|
|
time.sleep(2)
|
|
expect.append(send_num)
|
|
load = {"tgt_type": "glob", "tgt": "*", "jid": send_num}
|
|
with patch(
|
|
"salt.utils.minions.CkMinions.check_minions",
|
|
MagicMock(
|
|
return_value={
|
|
"minions": ["minion"],
|
|
"missing": [],
|
|
"ssh_minions": False,
|
|
}
|
|
),
|
|
):
|
|
server_channel.publish(load)
|
|
server_channel.publish({"tgt_type": "glob", "tgt": "*", "stop": True})
|
|
gather.join()
|
|
server_channel.pub_close()
|
|
assert len(results) == send_num, (len(results), set(expect).difference(results))
|
|
|
|
@slowTest
|
|
def test_publish_to_pubserv_tcp(self):
|
|
"""
|
|
Test sending 10K messags to ZeroMQPubServerChannel using TCP transport
|
|
"""
|
|
opts = dict(self.master_config, ipc_mode="tcp", pub_hwm=0)
|
|
server_channel = salt.transport.zeromq.ZeroMQPubServerChannel(opts)
|
|
server_channel.pre_fork(
|
|
self.process_manager,
|
|
kwargs={"log_queue": salt.log.setup.get_multiprocessing_logging_queue()},
|
|
)
|
|
pub_uri = "tcp://{interface}:{publish_port}".format(**server_channel.opts)
|
|
send_num = 10000
|
|
expect = []
|
|
results = []
|
|
gather = threading.Thread(
|
|
target=self._gather_results, args=(self.minion_config, pub_uri, results,)
|
|
)
|
|
gather.start()
|
|
# Allow time for server channel to start, especially on windows
|
|
time.sleep(2)
|
|
for i in range(send_num):
|
|
expect.append(i)
|
|
load = {"tgt_type": "glob", "tgt": "*", "jid": i}
|
|
server_channel.publish(load)
|
|
gather.join()
|
|
server_channel.pub_close()
|
|
assert len(results) == send_num, (len(results), set(expect).difference(results))
|
|
|
|
@staticmethod
|
|
def _send_small(opts, sid, num=10):
|
|
server_channel = salt.transport.zeromq.ZeroMQPubServerChannel(opts)
|
|
for i in range(num):
|
|
load = {"tgt_type": "glob", "tgt": "*", "jid": "{}-{}".format(sid, i)}
|
|
server_channel.publish(load)
|
|
server_channel.pub_close()
|
|
|
|
@staticmethod
|
|
def _send_large(opts, sid, num=10, size=250000 * 3):
|
|
server_channel = salt.transport.zeromq.ZeroMQPubServerChannel(opts)
|
|
for i in range(num):
|
|
load = {
|
|
"tgt_type": "glob",
|
|
"tgt": "*",
|
|
"jid": "{}-{}".format(sid, i),
|
|
"xdata": "0" * size,
|
|
}
|
|
server_channel.publish(load)
|
|
server_channel.pub_close()
|
|
|
|
@skipIf(salt.utils.platform.is_freebsd(), "Skip on FreeBSD")
|
|
@slowTest
|
|
def test_issue_36469_tcp(self):
|
|
"""
|
|
Test sending both large and small messags to publisher using TCP
|
|
|
|
https://github.com/saltstack/salt/issues/36469
|
|
"""
|
|
opts = dict(self.master_config, ipc_mode="tcp", pub_hwm=0)
|
|
server_channel = salt.transport.zeromq.ZeroMQPubServerChannel(opts)
|
|
server_channel.pre_fork(
|
|
self.process_manager,
|
|
kwargs={"log_queue": salt.log.setup.get_multiprocessing_logging_queue()},
|
|
)
|
|
send_num = 10 * 4
|
|
expect = []
|
|
results = []
|
|
pub_uri = "tcp://{interface}:{publish_port}".format(**opts)
|
|
# Allow time for server channel to start, especially on windows
|
|
time.sleep(2)
|
|
gather = threading.Thread(
|
|
target=self._gather_results, args=(self.minion_config, pub_uri, results,)
|
|
)
|
|
gather.start()
|
|
with ThreadPoolExecutor(max_workers=4) as executor:
|
|
executor.submit(self._send_small, opts, 1)
|
|
executor.submit(self._send_small, opts, 2)
|
|
executor.submit(self._send_small, opts, 3)
|
|
executor.submit(self._send_large, opts, 4)
|
|
expect = ["{}-{}".format(a, b) for a in range(10) for b in (1, 2, 3, 4)]
|
|
time.sleep(0.1)
|
|
server_channel.publish({"tgt_type": "glob", "tgt": "*", "stop": True})
|
|
gather.join()
|
|
server_channel.pub_close()
|
|
assert len(results) == send_num, (len(results), set(expect).difference(results))
|
|
|
|
|
|
class AsyncZeroMQReqChannelTests(TestCase):
|
|
def test_force_close_all_instances(self):
|
|
zmq1 = MagicMock()
|
|
zmq2 = MagicMock()
|
|
zmq3 = MagicMock()
|
|
zmq_objects = {"zmq": {"1": zmq1, "2": zmq2}, "other_zmq": {"3": zmq3}}
|
|
|
|
with patch(
|
|
"salt.transport.zeromq.AsyncZeroMQReqChannel.instance_map", zmq_objects
|
|
):
|
|
salt.transport.zeromq.AsyncZeroMQReqChannel.force_close_all_instances()
|
|
|
|
self.assertEqual(zmq1.mock_calls, [call.close()])
|
|
self.assertEqual(zmq2.mock_calls, [call.close()])
|
|
self.assertEqual(zmq3.mock_calls, [call.close()])
|
|
|
|
# check if instance map changed
|
|
self.assertIs(
|
|
zmq_objects, salt.transport.zeromq.AsyncZeroMQReqChannel.instance_map
|
|
)
|