From 84d58f717b9e8fb71360ee4d4427f3ff35fe6389 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 20 Sep 2023 14:33:32 -0700 Subject: [PATCH 01/26] Do not allow duplicate message ids --- salt/transport/tcp.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py index ddde882e764..fb2967201e2 100644 --- a/salt/transport/tcp.py +++ b/salt/transport/tcp.py @@ -15,6 +15,7 @@ import queue import socket import threading import urllib +import uuid import salt.ext.tornado import salt.ext.tornado.concurrent @@ -569,10 +570,7 @@ class MessageClient: self.io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current() with salt.utils.asynchronous.current_ioloop(self.io_loop): self._tcp_client = TCPClientKeepAlive(opts, resolver=resolver) - self._mid = 1 - self._max_messages = int((1 << 31) - 2) # number of IDs before we wrap # TODO: max queue size - self.send_queue = [] # queue of messages to be sent self.send_future_map = {} # mapping of request_id -> Future self._read_until_future = None @@ -722,18 +720,7 @@ class MessageClient: self._stream_return_running = False def _message_id(self): - wrap = False - while self._mid in self.send_future_map: - if self._mid >= self._max_messages: - if wrap: - # this shouldn't ever happen, but just in case - raise Exception("Unable to find available messageid") - self._mid = 1 - wrap = True - else: - self._mid += 1 - - return self._mid + return str(uuid.uuid4()) # TODO: return a message object which takes care of multiplexing? def on_recv(self, callback): From 529d27061e9b80dfd1b18dbb794d5511e3d4be13 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 20 Sep 2023 14:57:14 -0700 Subject: [PATCH 02/26] Use closure to preserve message future --- salt/transport/zeromq.py | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index ad7c66f49fb..dc67d3aca6b 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -519,6 +519,7 @@ class AsyncReqMessageClient: self._closing = False self._future = None + self._send_future_map = {} self.lock = salt.ext.tornado.locks.Lock() def connect(self): @@ -577,18 +578,6 @@ class AsyncReqMessageClient: self.stream = zmq.eventloop.zmqstream.ZMQStream( self.socket, io_loop=self.io_loop ) - self.stream.on_recv(self.handle_reply) - - def timeout_message(self, future): - """ - Handle a message timeout by removing it from the sending queue - and informing the caller - - :raises: SaltReqTimeoutError - """ - if self._future == future: - self._future = None - future.set_exception(SaltReqTimeoutError("Message timed out")) @salt.ext.tornado.gen.coroutine def send(self, message, timeout=None, callback=None): @@ -610,22 +599,26 @@ class AsyncReqMessageClient: if self.opts.get("detect_mode") is True: timeout = 1 + def timeout_message(future): + if not future.done(): + future.set_exception(SaltReqTimeoutError("Message timed out")) + if timeout is not None: send_timeout = self.io_loop.call_later( - timeout, self.timeout_message, future + timeout, timeout_message, future ) + def mark_future(msg): + if not future.done(): + data = salt.payload.loads(msg[0]) + future.set_result(data) + with (yield self.lock.acquire()): - self._future = future + self.stream.on_recv(mark_future) yield self.stream.send(message) recv = yield future - raise salt.ext.tornado.gen.Return(recv) - def handle_reply(self, msg): - data = salt.payload.loads(msg[0]) - future = self._future - self._future = None - future.set_result(data) + raise salt.ext.tornado.gen.Return(recv) class ZeroMQSocketMonitor: From f6ad1a5f9476cf07c7369a14477f6ee2ced6418c Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 21 Sep 2023 13:56:48 -0700 Subject: [PATCH 03/26] Fix timeout test --- tests/pytests/unit/transport/test_zeromq.py | 23 ++++++--------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/tests/pytests/unit/transport/test_zeromq.py b/tests/pytests/unit/transport/test_zeromq.py index e9149d5d546..04b0577b916 100644 --- a/tests/pytests/unit/transport/test_zeromq.py +++ b/tests/pytests/unit/transport/test_zeromq.py @@ -1480,20 +1480,9 @@ async def test_client_timeout_msg(minion_opts): client = salt.transport.zeromq.AsyncReqMessageClient( minion_opts, "tcp://127.0.0.1:4506" ) - assert hasattr(client, "_future") - assert client._future is None - future = salt.ext.tornado.concurrent.Future() - client._future = future - client.timeout_message(future) - with pytest.raises(salt.exceptions.SaltReqTimeoutError): - await future - assert client._future is None - - future_a = salt.ext.tornado.concurrent.Future() - future_b = salt.ext.tornado.concurrent.Future() - future_b.set_exception = MagicMock() - client._future = future_a - client.timeout_message(future_b) - - assert client._future == future_a - future_b.set_exception.assert_not_called() + client.connect() + try: + with pytest.raises(salt.exceptions.SaltReqTimeoutError): + await client.send({"meh": "bah"}, 1) + finally: + client.close() From 8a872eff085a7d6fed21e6a18e5c07382ced7698 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 27 Sep 2023 18:00:41 -0700 Subject: [PATCH 04/26] Preserve futures with messages This also make the message client behave more like it does after having been refactored to use native asyncio coroutines. --- salt/transport/zeromq.py | 61 ++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 37 deletions(-) diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index dc67d3aca6b..1a99627310c 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -12,6 +12,7 @@ from random import randint import zmq.error import zmq.eventloop.zmqstream +import zmq.eventloop.future import salt.ext.tornado import salt.ext.tornado.concurrent @@ -513,17 +514,17 @@ class AsyncReqMessageClient: else: self.io_loop = io_loop - self.context = zmq.Context() + self.context = zmq.eventloop.future.Context() self.send_queue = [] self._closing = False - self._future = None self._send_future_map = {} self.lock = salt.ext.tornado.locks.Lock() + self.ident = threading.get_ident() def connect(self): - if hasattr(self, "stream"): + if hasattr(self, "socket") and self.socket: return # wire up sockets self._init_socket() @@ -539,24 +540,10 @@ class AsyncReqMessageClient: return else: self._closing = True - if hasattr(self, "stream") and self.stream is not None: - if ZMQ_VERSION_INFO < (14, 3, 0): - # stream.close() doesn't work properly on pyzmq < 14.3.0 - if self.stream.socket: - self.stream.socket.close() - self.stream.io_loop.remove_handler(self.stream.socket) - # set this to None, more hacks for messed up pyzmq - self.stream.socket = None - self.socket.close() - else: - self.stream.close(1) - self.socket = None - self.stream = None - if self._future: - self._future.set_exception(SaltException("Closing connection")) - self._future = None + if hasattr(self, "socket") and self.socket is not None: + self.socket.close(0) + self.socket = None if self.context.closed is False: - # This hangs if closing the stream causes an import error self.context.term() def _init_socket(self): @@ -573,11 +560,8 @@ class AsyncReqMessageClient: self.socket.setsockopt(zmq.IPV6, 1) elif hasattr(zmq, "IPV4ONLY"): self.socket.setsockopt(zmq.IPV4ONLY, 0) - self.socket.linger = self.linger + self.socket.setsockopt(zmq.LINGER, self.linger) self.socket.connect(self.addr) - self.stream = zmq.eventloop.zmqstream.ZMQStream( - self.socket, io_loop=self.io_loop - ) @salt.ext.tornado.gen.coroutine def send(self, message, timeout=None, callback=None): @@ -599,27 +583,30 @@ class AsyncReqMessageClient: if self.opts.get("detect_mode") is True: timeout = 1 - def timeout_message(future): - if not future.done(): - future.set_exception(SaltReqTimeoutError("Message timed out")) - if timeout is not None: send_timeout = self.io_loop.call_later( - timeout, timeout_message, future + timeout, self._timeout_message, future ) - def mark_future(msg): - if not future.done(): - data = salt.payload.loads(msg[0]) - future.set_result(data) + self.io_loop.spawn_callback(self._send_recv, message, future) - with (yield self.lock.acquire()): - self.stream.on_recv(mark_future) - yield self.stream.send(message) - recv = yield future + recv = yield future raise salt.ext.tornado.gen.Return(recv) + def _timeout_message(self, future): + if not future.done(): + future.set_exception(SaltReqTimeoutError("Message timed out")) + + @salt.ext.tornado.gen.coroutine + def _send_recv(self, message, future): + with (yield self.lock.acquire()): + yield self.socket.send(message) + recv = yield self.socket.recv() + if not future.done(): + data = salt.payload.loads(recv) + future.set_result(data) + class ZeroMQSocketMonitor: __EVENT_MAP = None From ae1bf35ce803998eed9583d4fb69b9caa78420e2 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 11 Oct 2023 00:10:22 -0700 Subject: [PATCH 05/26] Add test coverage --- salt/transport/tcp.py | 182 ++++++++--------- salt/transport/zeromq.py | 2 +- .../pytests/functional/channel/test_server.py | 1 + .../tcp/test_load_balanced_server.py | 55 ++++++ .../transport/tcp/test_pub_server.py | 58 ++++++ .../zeromq/test_pub_server_channel.py | 86 ++++++++ .../transport/zeromq/test_request_client.py | 6 +- tests/pytests/unit/transport/test_tcp.py | 185 +++++++++++++++++- tests/pytests/unit/transport/test_zeromq.py | 12 ++ 9 files changed, 486 insertions(+), 101 deletions(-) create mode 100644 tests/pytests/functional/transport/tcp/test_load_balanced_server.py create mode 100644 tests/pytests/functional/transport/tcp/test_pub_server.py diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py index fb2967201e2..81454d0eab5 100644 --- a/salt/transport/tcp.py +++ b/salt/transport/tcp.py @@ -35,6 +35,7 @@ import salt.utils.platform import salt.utils.versions from salt.exceptions import SaltClientError, SaltReqTimeoutError from salt.utils.network import ip_bracket +from salt.utils.process import SignalHandlingProcess if salt.utils.platform.is_windows(): USE_LOAD_BALANCER = True @@ -43,7 +44,6 @@ else: if USE_LOAD_BALANCER: import salt.ext.tornado.util - from salt.utils.process import SignalHandlingProcess log = logging.getLogger(__name__) @@ -128,69 +128,64 @@ def _set_tcp_keepalive(sock, opts): sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0) -if USE_LOAD_BALANCER: +class LoadBalancerServer(SignalHandlingProcess): + """ + Raw TCP server which runs in its own process and will listen + for incoming connections. Each incoming connection will be + sent via multiprocessing queue to the workers. + Since the queue is shared amongst workers, only one worker will + handle a given connection. + """ - class LoadBalancerServer(SignalHandlingProcess): - """ - Raw TCP server which runs in its own process and will listen - for incoming connections. Each incoming connection will be - sent via multiprocessing queue to the workers. - Since the queue is shared amongst workers, only one worker will - handle a given connection. - """ + # TODO: opts! + # Based on default used in salt.ext.tornado.netutil.bind_sockets() + backlog = 128 - # TODO: opts! - # Based on default used in salt.ext.tornado.netutil.bind_sockets() - backlog = 128 + def __init__(self, opts, socket_queue, **kwargs): + super().__init__(**kwargs) + self.opts = opts + self.socket_queue = socket_queue + self._socket = None - def __init__(self, opts, socket_queue, **kwargs): - super().__init__(**kwargs) - self.opts = opts - self.socket_queue = socket_queue + def close(self): + if self._socket is not None: + self._socket.shutdown(socket.SHUT_RDWR) + self._socket.close() self._socket = None - def close(self): - if self._socket is not None: - self._socket.shutdown(socket.SHUT_RDWR) - self._socket.close() - self._socket = None + # pylint: disable=W1701 + def __del__(self): + self.close() - # pylint: disable=W1701 - def __del__(self): - self.close() + # pylint: enable=W1701 - # pylint: enable=W1701 + def run(self): + """ + Start the load balancer + """ + self._socket = _get_socket(self.opts) + self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + _set_tcp_keepalive(self._socket, self.opts) + self._socket.setblocking(1) + self._socket.bind(_get_bind_addr(self.opts, "ret_port")) + self._socket.listen(self.backlog) - def run(self): - """ - Start the load balancer - """ - self._socket = _get_socket(self.opts) - self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - _set_tcp_keepalive(self._socket, self.opts) - self._socket.setblocking(1) - self._socket.bind(_get_bind_addr(self.opts, "ret_port")) - self._socket.listen(self.backlog) - - while True: - try: - # Wait for a connection to occur since the socket is - # blocking. - connection, address = self._socket.accept() - # Wait for a free slot to be available to put - # the connection into. - # Sockets are picklable on Windows in Python 3. - self.socket_queue.put((connection, address), True, None) - except OSError as e: - # ECONNABORTED indicates that there was a connection - # but it was closed while still in the accept queue. - # (observed on FreeBSD). - if ( - salt.ext.tornado.util.errno_from_exception(e) - == errno.ECONNABORTED - ): - continue - raise + while True: + try: + # Wait for a connection to occur since the socket is + # blocking. + connection, address = self._socket.accept() + # Wait for a free slot to be available to put + # the connection into. + # Sockets are picklable on Windows in Python 3. + self.socket_queue.put((connection, address), True, None) + except OSError as e: + # ECONNABORTED indicates that there was a connection + # but it was closed while still in the accept queue. + # (observed on FreeBSD). + if salt.ext.tornado.util.errno_from_exception(e) == errno.ECONNABORTED: + continue + raise class Resolver: @@ -468,45 +463,43 @@ class SaltMessageServer(salt.ext.tornado.tcpserver.TCPServer): raise -if USE_LOAD_BALANCER: +class LoadBalancerWorker(SaltMessageServer): + """ + This will receive TCP connections from 'LoadBalancerServer' via + a multiprocessing queue. + Since the queue is shared amongst workers, only one worker will handle + a given connection. + """ - class LoadBalancerWorker(SaltMessageServer): - """ - This will receive TCP connections from 'LoadBalancerServer' via - a multiprocessing queue. - Since the queue is shared amongst workers, only one worker will handle - a given connection. - """ + def __init__(self, socket_queue, message_handler, *args, **kwargs): + super().__init__(message_handler, *args, **kwargs) + self.socket_queue = socket_queue + self._stop = threading.Event() + self.thread = threading.Thread(target=self.socket_queue_thread) + self.thread.start() - def __init__(self, socket_queue, message_handler, *args, **kwargs): - super().__init__(message_handler, *args, **kwargs) - self.socket_queue = socket_queue - self._stop = threading.Event() - self.thread = threading.Thread(target=self.socket_queue_thread) - self.thread.start() + def close(self): + self._stop.set() + self.thread.join() + super().close() - def close(self): - self._stop.set() - self.thread.join() - super().close() - - def socket_queue_thread(self): - try: - while True: - try: - client_socket, address = self.socket_queue.get(True, 1) - except queue.Empty: - if self._stop.is_set(): - break - continue - # 'self.io_loop' initialized in super class - # 'salt.ext.tornado.tcpserver.TCPServer'. - # 'self._handle_connection' defined in same super class. - self.io_loop.spawn_callback( - self._handle_connection, client_socket, address - ) - except (KeyboardInterrupt, SystemExit): - pass + def socket_queue_thread(self): + try: + while True: + try: + client_socket, address = self.socket_queue.get(True, 1) + except queue.Empty: + if self._stop.is_set(): + break + continue + # 'self.io_loop' initialized in super class + # 'salt.ext.tornado.tcpserver.TCPServer'. + # 'self._handle_connection' defined in same super class. + self.io_loop.spawn_callback( + self._handle_connection, client_socket, address + ) + except (KeyboardInterrupt, SystemExit): + pass class TCPClientKeepAlive(salt.ext.tornado.tcpclient.TCPClient): @@ -583,10 +576,6 @@ class MessageClient: self.backoff = opts.get("tcp_reconnect_backoff", 1) - def _stop_io_loop(self): - if self.io_loop is not None: - self.io_loop.stop() - # TODO: timeout inflight sessions def close(self): if self._closing: @@ -962,6 +951,7 @@ class TCPPublishServer(salt.transport.base.DaemonizedPublishServer): """ io_loop = salt.ext.tornado.ioloop.IOLoop() io_loop.make_current() + self.io_loop = io_loop # Spin up the publisher self.pub_server = pub_server = PubServer( diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index 1a99627310c..54b8bf47ba7 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -11,8 +11,8 @@ import threading from random import randint import zmq.error -import zmq.eventloop.zmqstream import zmq.eventloop.future +import zmq.eventloop.zmqstream import salt.ext.tornado import salt.ext.tornado.concurrent diff --git a/tests/pytests/functional/channel/test_server.py b/tests/pytests/functional/channel/test_server.py index 82a9450450f..da6f9caa8c9 100644 --- a/tests/pytests/functional/channel/test_server.py +++ b/tests/pytests/functional/channel/test_server.py @@ -164,6 +164,7 @@ def test_pub_server_channel( log.info("TEST - Req Server handle payload %r", payload) req_server_channel.post_fork(handle_payload, io_loop=io_loop) + if master_config["transport"] == "zeromq": time.sleep(1) attempts = 5 diff --git a/tests/pytests/functional/transport/tcp/test_load_balanced_server.py b/tests/pytests/functional/transport/tcp/test_load_balanced_server.py new file mode 100644 index 00000000000..cfc25f917e5 --- /dev/null +++ b/tests/pytests/functional/transport/tcp/test_load_balanced_server.py @@ -0,0 +1,55 @@ +import multiprocessing +import socket +import threading +import time + +import pytest + +import salt.transport.tcp + +pytestmark = [ + pytest.mark.core_test, +] + + +def test_tcp_load_balancer_server(master_opts, io_loop): + + messages = [] + + def handler(stream, message, header): + messages.append(message) + + queue = multiprocessing.Queue() + server = salt.transport.tcp.LoadBalancerServer(master_opts, queue) + worker = salt.transport.tcp.LoadBalancerWorker(queue, handler, io_loop=io_loop) + + def run_loop(): + io_loop.start() + + loop_thread = threading.Thread(target=run_loop) + loop_thread.start() + + thread = threading.Thread(target=server.run) + thread.start() + + # Wait for bind to happen. + time.sleep(0.5) + + package = {"foo": "bar"} + payload = salt.transport.frame.frame_msg(package) + sock = socket.socket() + sock.connect(("127.0.0.1", master_opts["ret_port"])) + sock.send(payload) + + try: + start = time.monotonic() + while not messages: + time.sleep(0.3) + if time.monotonic() - start > 30: + assert False, "Took longer than 30 seconds to receive message" + assert [package] == messages + finally: + server.close() + thread.join() + io_loop.stop() + worker.close() diff --git a/tests/pytests/functional/transport/tcp/test_pub_server.py b/tests/pytests/functional/transport/tcp/test_pub_server.py new file mode 100644 index 00000000000..a25b5385138 --- /dev/null +++ b/tests/pytests/functional/transport/tcp/test_pub_server.py @@ -0,0 +1,58 @@ +import threading +import time + +import salt.ext.tornado.gen +import salt.transport.tcp + + +async def test_pub_channel(master_opts, minion_opts, io_loop): + def presence_callback(client): + pass + + def remove_presence_callback(client): + pass + + master_opts["transport"] = "tcp" + minion_opts.update(master_ip="127.0.0.1", transport="tcp") + + server = salt.transport.tcp.TCPPublishServer(master_opts) + + client = salt.transport.tcp.TCPPubClient(minion_opts, io_loop) + + payloads = [] + + publishes = [] + + def publish_payload(payload, callback): + server.publish_payload(payload) + payloads.append(payload) + + def on_recv(message): + print("ON RECV") + publishes.append(message) + + thread = threading.Thread( + target=server.publish_daemon, + args=(publish_payload, presence_callback, remove_presence_callback), + ) + thread.start() + + # Wait for socket to bind. + time.sleep(3) + + await client.connect(master_opts["publish_port"]) + client.on_recv(on_recv) + + print("Publish message") + server.publish({"meh": "bah"}) + + start = time.monotonic() + try: + while not publishes: + await salt.ext.tornado.gen.sleep(0.3) + if time.monotonic() - start > 30: + assert False, "Message not published after 30 seconds" + finally: + server.io_loop.stop() + thread.join() + server.io_loop.close(all_fds=True) diff --git a/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py b/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py index 3e46ab1b6a1..27a315fda91 100644 --- a/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py +++ b/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py @@ -1,7 +1,10 @@ import logging +import threading +import time import pytest +import salt.transport.zeromq from tests.support.mock import MagicMock, patch from tests.support.pytest.transport import PubServerChannelProcess @@ -51,3 +54,86 @@ def test_zeromq_filtering(salt_master, salt_minion): assert len(results) == send_num, "{} != {}, difference: {}".format( len(results), send_num, set(expect).difference(results) ) + + +def test_pub_channel(master_opts): + server = salt.transport.zeromq.PublishServer(master_opts) + + payloads = [] + + def publish_payload(payload): + server.publish_payload(payload) + payloads.append(payload) + + thread = threading.Thread(target=server.publish_daemon, args=(publish_payload,)) + thread.start() + + server.publish({"meh": "bah"}) + + start = time.monotonic() + try: + while not payloads: + time.sleep(0.3) + if time.monotonic() - start > 30: + assert False, "No message received after 30 seconds" + finally: + server.close() + server.io_loop.stop() + thread.join() + server.io_loop.close(all_fds=True) + + +def test_pub_channel_filtering(master_opts): + master_opts["zmq_filtering"] = True + server = salt.transport.zeromq.PublishServer(master_opts) + + payloads = [] + + def publish_payload(payload): + server.publish_payload(payload) + payloads.append(payload) + + thread = threading.Thread(target=server.publish_daemon, args=(publish_payload,)) + thread.start() + + server.publish({"meh": "bah"}) + + start = time.monotonic() + try: + while not payloads: + time.sleep(0.3) + if time.monotonic() - start > 30: + assert False, "No message received after 30 seconds" + finally: + server.close() + server.io_loop.stop() + thread.join() + server.io_loop.close(all_fds=True) + + +def test_pub_channel_filtering_topic(master_opts): + master_opts["zmq_filtering"] = True + server = salt.transport.zeromq.PublishServer(master_opts) + + payloads = [] + + def publish_payload(payload): + server.publish_payload(payload, topic_list=["meh"]) + payloads.append(payload) + + thread = threading.Thread(target=server.publish_daemon, args=(publish_payload,)) + thread.start() + + server.publish({"meh": "bah"}) + + start = time.monotonic() + try: + while not payloads: + time.sleep(0.3) + if time.monotonic() - start > 30: + assert False, "No message received after 30 seconds" + finally: + server.close() + server.io_loop.stop() + thread.join() + server.io_loop.close(all_fds=True) diff --git a/tests/pytests/functional/transport/zeromq/test_request_client.py b/tests/pytests/functional/transport/zeromq/test_request_client.py index 7ab2295baaf..4ee99f49aa3 100644 --- a/tests/pytests/functional/transport/zeromq/test_request_client.py +++ b/tests/pytests/functional/transport/zeromq/test_request_client.py @@ -32,8 +32,8 @@ async def test_request_channel_issue_64627(io_loop, minion_opts, port): request_client = salt.transport.zeromq.RequestClient(minion_opts, io_loop) rep = await request_client.send(b"foo") - req_socket = request_client.message_client.stream.socket + req_socket = request_client.message_client.socket rep = await request_client.send(b"foo") - assert req_socket is request_client.message_client.stream.socket + assert req_socket is request_client.message_client.socket request_client.close() - assert request_client.message_client.stream is None + assert request_client.message_client.socket is None diff --git a/tests/pytests/unit/transport/test_tcp.py b/tests/pytests/unit/transport/test_tcp.py index bcfb71f5590..17c7e072749 100644 --- a/tests/pytests/unit/transport/test_tcp.py +++ b/tests/pytests/unit/transport/test_tcp.py @@ -9,10 +9,11 @@ from pytestshellutils.utils import ports import salt.channel.server import salt.exceptions import salt.ext.tornado +import salt.ext.tornado.concurrent import salt.transport.tcp from tests.support.mock import MagicMock, PropertyMock, patch -pytestmark = [ +tpytestmark = [ pytest.mark.core_test, ] @@ -483,3 +484,185 @@ def test_presence_removed_on_stream_closed(): io_loop.run_sync(functools.partial(server.publish_payload, package, None)) server.remove_presence_callback.assert_called_with(client) + + +async def test_tcp_pub_client_decode_dict(minion_opts, io_loop): + dmsg = {"meh": "bah"} + client = salt.transport.tcp.TCPPubClient(minion_opts, io_loop) + assert dmsg == await client._decode_messages(dmsg) + + +async def test_tcp_pub_client_decode_msgpack(minion_opts, io_loop): + dmsg = {"meh": "bah"} + msg = salt.payload.dumps(dmsg) + client = salt.transport.tcp.TCPPubClient(minion_opts, io_loop) + assert dmsg == await client._decode_messages(msg) + + +def test_tcp_pub_client_close(minion_opts, io_loop): + client = salt.transport.tcp.TCPPubClient(minion_opts, io_loop) + + message_client = MagicMock() + + client.message_client = message_client + client.close() + assert client._closing is True + assert client.message_client is None + client.close() + message_client.close.assert_called_once_with() + + +async def test_pub_server__stream_read(master_opts, io_loop): + + messages = [salt.transport.frame.frame_msg({"foo": "bar"})] + + class Stream: + def __init__(self, messages): + self.messages = messages + + def read_bytes(self, *args, **kwargs): + if self.messages: + msg = self.messages.pop(0) + future = salt.ext.tornado.concurrent.Future() + future.set_result(msg) + return future + raise salt.ext.tornado.iostream.StreamClosedError() + + client = MagicMock() + client.stream = Stream(messages) + client.address = "client address" + server = salt.transport.tcp.PubServer(master_opts, io_loop) + await server._stream_read(client) + client.close.assert_called_once() + + +async def test_pub_server__stream_read_exception(master_opts, io_loop): + client = MagicMock() + client.stream = MagicMock() + client.stream.read_bytes = MagicMock( + side_effect=[ + Exception("Something went wrong"), + salt.ext.tornado.iostream.StreamClosedError(), + ] + ) + client.address = "client address" + server = salt.transport.tcp.PubServer(master_opts, io_loop) + await server._stream_read(client) + client.close.assert_called_once() + + +async def test_salt_message_server(master_opts): + + received = [] + + def handler(stream, body, header): + + received.append(body) + + server = salt.transport.tcp.SaltMessageServer(handler) + msg = {"foo": "bar"} + messages = [salt.transport.frame.frame_msg(msg)] + + class Stream: + def __init__(self, messages): + self.messages = messages + + def read_bytes(self, *args, **kwargs): + if self.messages: + msg = self.messages.pop(0) + future = salt.ext.tornado.concurrent.Future() + future.set_result(msg) + return future + raise salt.ext.tornado.iostream.StreamClosedError() + + stream = Stream(messages) + address = "client address" + + await server.handle_stream(stream, address) + + # Let loop iterate so callback gets called + await salt.ext.tornado.gen.sleep(0.01) + + assert received + assert [msg] == received + + +async def test_salt_message_server_exception(master_opts, io_loop): + received = [] + + def handler(stream, body, header): + + received.append(body) + + stream = MagicMock() + stream.read_bytes = MagicMock( + side_effect=[ + Exception("Something went wrong"), + ] + ) + address = "client address" + server = salt.transport.tcp.SaltMessageServer(handler) + await server.handle_stream(stream, address) + stream.close.assert_called_once() + + +async def test_message_client_stream_return_exception(minion_opts, io_loop): + msg = {"foo": "bar"} + payload = salt.transport.frame.frame_msg(msg) + future = salt.ext.tornado.concurrent.Future() + future.set_result(payload) + client = salt.transport.tcp.MessageClient( + minion_opts, + "127.0.0.1", + 12345, + connect_callback=MagicMock(), + disconnect_callback=MagicMock(), + ) + client._stream = MagicMock() + client._stream.read_bytes.side_effect = [ + future, + ] + try: + io_loop.add_callback(client._stream_return) + await salt.ext.tornado.gen.sleep(0.01) + client.close() + await salt.ext.tornado.gen.sleep(0.01) + assert client._stream is None + finally: + client.close() + + +def test_tcp_pub_server_pre_fork(master_opts): + process_manager = MagicMock() + server = salt.transport.tcp.TCPPublishServer(master_opts) + server.pre_fork(process_manager) + + +async def test_pub_server_publish_payload(master_opts, io_loop): + server = salt.transport.tcp.PubServer(master_opts, io_loop=io_loop) + package = {"foo": "bar"} + topic_list = ["meh"] + future = salt.ext.tornado.concurrent.Future() + future.set_result(None) + client = MagicMock() + client.stream = MagicMock() + client.stream.write.side_effect = [future] + client.id_ = "meh" + server.clients = [client] + await server.publish_payload(package, topic_list) + client.stream.write.assert_called_once() + + +async def test_pub_server_publish_payload_closed_stream(master_opts, io_loop): + server = salt.transport.tcp.PubServer(master_opts, io_loop=io_loop) + package = {"foo": "bar"} + topic_list = ["meh"] + client = MagicMock() + client.stream = MagicMock() + client.stream.write.side_effect = [ + salt.ext.tornado.iostream.StreamClosedError("mock") + ] + client.id_ = "meh" + server.clients = {client} + await server.publish_payload(package, topic_list) + assert server.clients == set() diff --git a/tests/pytests/unit/transport/test_zeromq.py b/tests/pytests/unit/transport/test_zeromq.py index 04b0577b916..2bad5f9ae5f 100644 --- a/tests/pytests/unit/transport/test_zeromq.py +++ b/tests/pytests/unit/transport/test_zeromq.py @@ -1414,6 +1414,7 @@ async def test_req_server_garbage_request(io_loop): RequestServers's message handler. """ opts = salt.config.master_config("") + opts["zmq_monitor"] = True request_server = salt.transport.zeromq.RequestServer(opts) def message_handler(payload): @@ -1486,3 +1487,14 @@ async def test_client_timeout_msg(minion_opts): await client.send({"meh": "bah"}, 1) finally: client.close() + + +def test_pub_client_init(minion_opts, io_loop): + minion_opts["id"] = "minion" + minion_opts["__role"] = "syndic" + minion_opts["master_ip"] = "127.0.0.1" + minion_opts["zmq_filtering"] = True + minion_opts["zmq_monitor"] = True + client = salt.transport.zeromq.PublishClient(minion_opts, io_loop) + client.send(b"asf") + client.close() From 096314341f92fba9ece44aa3835cd817aa8bd26f Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 25 Oct 2023 15:36:07 -0700 Subject: [PATCH 06/26] Add changelog --- changelog/65114.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/65114.fixed.md diff --git a/changelog/65114.fixed.md b/changelog/65114.fixed.md new file mode 100644 index 00000000000..fb4f2ab1536 --- /dev/null +++ b/changelog/65114.fixed.md @@ -0,0 +1 @@ +Fix nonce verification, request server replies do not stomp on eachother. From 7df5e0a4d5afef18bebc8390162970434fc3d48e Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 27 Oct 2023 18:16:33 +0100 Subject: [PATCH 07/26] Always download the onedir artifact. Otherwise the publish repositories step won't happen if we skip the package download tests. Signed-off-by: Pedro Algarvio --- .github/workflows/release.yml | 1 - .github/workflows/templates/release.yml.jinja | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 71dfb5dc50e..0a31e7601e2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -116,7 +116,6 @@ jobs: download-onedir-artifact: name: Download Staging Onedir Artifact - if: ${{ inputs.skip-salt-pkg-download-test-suite == false }} runs-on: - self-hosted - linux diff --git a/.github/workflows/templates/release.yml.jinja b/.github/workflows/templates/release.yml.jinja index c79fa26d02f..7c5c28af059 100644 --- a/.github/workflows/templates/release.yml.jinja +++ b/.github/workflows/templates/release.yml.jinja @@ -150,7 +150,6 @@ permissions: download-onedir-artifact: name: Download Staging Onedir Artifact - if: ${{ inputs.skip-salt-pkg-download-test-suite == false }} runs-on: - self-hosted - linux From 68bca1ee8dabe2a57c09786522989f949dac42dd Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 30 Oct 2023 21:51:00 +0000 Subject: [PATCH 08/26] On nightly and scheduled builds, don't run split tests. GH is not coping. Signed-off-by: Pedro Algarvio --- .github/workflows/ci.yml | 29 +++++++++++++++++ .github/workflows/nightly.yml | 31 ++++++++++++++++++- .github/workflows/scheduled.yml | 29 +++++++++++++++++ .github/workflows/staging.yml | 29 +++++++++++++++++ .github/workflows/templates/nightly.yml.jinja | 2 +- .../workflows/templates/test-salt.yml.jinja | 3 ++ .github/workflows/test-action-macos.yml | 7 ++++- .github/workflows/test-action.yml | 7 ++++- tools/ci.py | 10 +++++- 9 files changed, 142 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 450f9936681..92cb24cd349 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1847,6 +1847,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci windows-2019: name: Windows 2019 Test @@ -1867,6 +1868,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci windows-2022: name: Windows 2022 Test @@ -1887,6 +1889,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci macos-12: name: macOS 12 Test @@ -1907,6 +1910,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci almalinux-8: name: Alma Linux 8 Test @@ -1927,6 +1931,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci almalinux-9: name: Alma Linux 9 Test @@ -1947,6 +1952,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci amazonlinux-2: name: Amazon Linux 2 Test @@ -1967,6 +1973,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci archlinux-lts: name: Arch Linux LTS Test @@ -1987,6 +1994,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci centos-7: name: CentOS 7 Test @@ -2007,6 +2015,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci centosstream-8: name: CentOS Stream 8 Test @@ -2027,6 +2036,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci centosstream-9: name: CentOS Stream 9 Test @@ -2047,6 +2057,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci debian-10: name: Debian 10 Test @@ -2067,6 +2078,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci debian-11: name: Debian 11 Test @@ -2087,6 +2099,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci debian-11-arm64: name: Debian 11 Arm64 Test @@ -2107,6 +2120,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci debian-12: name: Debian 12 Test @@ -2127,6 +2141,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci debian-12-arm64: name: Debian 12 Arm64 Test @@ -2147,6 +2162,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci fedora-37: name: Fedora 37 Test @@ -2167,6 +2183,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci fedora-38: name: Fedora 38 Test @@ -2187,6 +2204,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci opensuse-15: name: Opensuse 15 Test @@ -2207,6 +2225,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci photonos-3: name: Photon OS 3 Test @@ -2227,6 +2246,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci photonos-3-arm64: name: Photon OS 3 Arm64 Test @@ -2247,6 +2267,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci photonos-4: name: Photon OS 4 Test @@ -2267,6 +2288,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci photonos-4-arm64: name: Photon OS 4 Arm64 Test @@ -2287,6 +2309,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci photonos-5: name: Photon OS 5 Test @@ -2307,6 +2330,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci photonos-5-arm64: name: Photon OS 5 Arm64 Test @@ -2327,6 +2351,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci ubuntu-2004: name: Ubuntu 20.04 Test @@ -2347,6 +2372,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci ubuntu-2004-arm64: name: Ubuntu 20.04 Arm64 Test @@ -2367,6 +2393,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci ubuntu-2204: name: Ubuntu 22.04 Test @@ -2387,6 +2414,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci ubuntu-2204-arm64: name: Ubuntu 22.04 Arm64 Test @@ -2407,6 +2435,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} + workflow-slug: ci combine-all-code-coverage: name: Combine Code Coverage diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 26e1f54df40..e15743ed459 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -18,7 +18,7 @@ on: description: Skip running the Salt packages test suite. schedule: # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onschedule - - cron: '0 1 * * *' # Every day at 1AM + - cron: '0 0 * * *' # Every day at 0AM env: COLUMNS: 190 @@ -1908,6 +1908,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly windows-2019: name: Windows 2019 Test @@ -1928,6 +1929,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly windows-2022: name: Windows 2022 Test @@ -1948,6 +1950,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly macos-12: name: macOS 12 Test @@ -1968,6 +1971,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly almalinux-8: name: Alma Linux 8 Test @@ -1988,6 +1992,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly almalinux-9: name: Alma Linux 9 Test @@ -2008,6 +2013,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly amazonlinux-2: name: Amazon Linux 2 Test @@ -2028,6 +2034,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly archlinux-lts: name: Arch Linux LTS Test @@ -2048,6 +2055,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly centos-7: name: CentOS 7 Test @@ -2068,6 +2076,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly centosstream-8: name: CentOS Stream 8 Test @@ -2088,6 +2097,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly centosstream-9: name: CentOS Stream 9 Test @@ -2108,6 +2118,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly debian-10: name: Debian 10 Test @@ -2128,6 +2139,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly debian-11: name: Debian 11 Test @@ -2148,6 +2160,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly debian-11-arm64: name: Debian 11 Arm64 Test @@ -2168,6 +2181,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly debian-12: name: Debian 12 Test @@ -2188,6 +2202,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly debian-12-arm64: name: Debian 12 Arm64 Test @@ -2208,6 +2223,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly fedora-37: name: Fedora 37 Test @@ -2228,6 +2244,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly fedora-38: name: Fedora 38 Test @@ -2248,6 +2265,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly opensuse-15: name: Opensuse 15 Test @@ -2268,6 +2286,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly photonos-3: name: Photon OS 3 Test @@ -2288,6 +2307,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly photonos-3-arm64: name: Photon OS 3 Arm64 Test @@ -2308,6 +2328,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly photonos-4: name: Photon OS 4 Test @@ -2328,6 +2349,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly photonos-4-arm64: name: Photon OS 4 Arm64 Test @@ -2348,6 +2370,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly photonos-5: name: Photon OS 5 Test @@ -2368,6 +2391,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly photonos-5-arm64: name: Photon OS 5 Arm64 Test @@ -2388,6 +2412,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly ubuntu-2004: name: Ubuntu 20.04 Test @@ -2408,6 +2433,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly ubuntu-2004-arm64: name: Ubuntu 20.04 Arm64 Test @@ -2428,6 +2454,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly ubuntu-2204: name: Ubuntu 22.04 Test @@ -2448,6 +2475,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly ubuntu-2204-arm64: name: Ubuntu 22.04 Arm64 Test @@ -2468,6 +2496,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: nightly combine-all-code-coverage: name: Combine Code Coverage diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 3a53d2d87b3..d8a80b2790b 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -1881,6 +1881,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled windows-2019: name: Windows 2019 Test @@ -1901,6 +1902,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled windows-2022: name: Windows 2022 Test @@ -1921,6 +1923,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled macos-12: name: macOS 12 Test @@ -1941,6 +1944,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled almalinux-8: name: Alma Linux 8 Test @@ -1961,6 +1965,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled almalinux-9: name: Alma Linux 9 Test @@ -1981,6 +1986,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled amazonlinux-2: name: Amazon Linux 2 Test @@ -2001,6 +2007,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled archlinux-lts: name: Arch Linux LTS Test @@ -2021,6 +2028,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled centos-7: name: CentOS 7 Test @@ -2041,6 +2049,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled centosstream-8: name: CentOS Stream 8 Test @@ -2061,6 +2070,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled centosstream-9: name: CentOS Stream 9 Test @@ -2081,6 +2091,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled debian-10: name: Debian 10 Test @@ -2101,6 +2112,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled debian-11: name: Debian 11 Test @@ -2121,6 +2133,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled debian-11-arm64: name: Debian 11 Arm64 Test @@ -2141,6 +2154,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled debian-12: name: Debian 12 Test @@ -2161,6 +2175,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled debian-12-arm64: name: Debian 12 Arm64 Test @@ -2181,6 +2196,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled fedora-37: name: Fedora 37 Test @@ -2201,6 +2217,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled fedora-38: name: Fedora 38 Test @@ -2221,6 +2238,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled opensuse-15: name: Opensuse 15 Test @@ -2241,6 +2259,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled photonos-3: name: Photon OS 3 Test @@ -2261,6 +2280,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled photonos-3-arm64: name: Photon OS 3 Arm64 Test @@ -2281,6 +2301,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled photonos-4: name: Photon OS 4 Test @@ -2301,6 +2322,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled photonos-4-arm64: name: Photon OS 4 Arm64 Test @@ -2321,6 +2343,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled photonos-5: name: Photon OS 5 Test @@ -2341,6 +2364,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled photonos-5-arm64: name: Photon OS 5 Arm64 Test @@ -2361,6 +2385,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled ubuntu-2004: name: Ubuntu 20.04 Test @@ -2381,6 +2406,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled ubuntu-2004-arm64: name: Ubuntu 20.04 Arm64 Test @@ -2401,6 +2427,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled ubuntu-2204: name: Ubuntu 22.04 Test @@ -2421,6 +2448,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled ubuntu-2204-arm64: name: Ubuntu 22.04 Arm64 Test @@ -2441,6 +2469,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: false skip-junit-reports: false + workflow-slug: scheduled combine-all-code-coverage: name: Combine Code Coverage diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index ed50f98cb22..4962220fe5a 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -1903,6 +1903,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging windows-2019: name: Windows 2019 Test @@ -1923,6 +1924,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging windows-2022: name: Windows 2022 Test @@ -1943,6 +1945,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging macos-12: name: macOS 12 Test @@ -1963,6 +1966,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging almalinux-8: name: Alma Linux 8 Test @@ -1983,6 +1987,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging almalinux-9: name: Alma Linux 9 Test @@ -2003,6 +2008,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging amazonlinux-2: name: Amazon Linux 2 Test @@ -2023,6 +2029,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging archlinux-lts: name: Arch Linux LTS Test @@ -2043,6 +2050,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging centos-7: name: CentOS 7 Test @@ -2063,6 +2071,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging centosstream-8: name: CentOS Stream 8 Test @@ -2083,6 +2092,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging centosstream-9: name: CentOS Stream 9 Test @@ -2103,6 +2113,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging debian-10: name: Debian 10 Test @@ -2123,6 +2134,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging debian-11: name: Debian 11 Test @@ -2143,6 +2155,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging debian-11-arm64: name: Debian 11 Arm64 Test @@ -2163,6 +2176,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging debian-12: name: Debian 12 Test @@ -2183,6 +2197,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging debian-12-arm64: name: Debian 12 Arm64 Test @@ -2203,6 +2218,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging fedora-37: name: Fedora 37 Test @@ -2223,6 +2239,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging fedora-38: name: Fedora 38 Test @@ -2243,6 +2260,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging opensuse-15: name: Opensuse 15 Test @@ -2263,6 +2281,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging photonos-3: name: Photon OS 3 Test @@ -2283,6 +2302,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging photonos-3-arm64: name: Photon OS 3 Arm64 Test @@ -2303,6 +2323,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging photonos-4: name: Photon OS 4 Test @@ -2323,6 +2344,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging photonos-4-arm64: name: Photon OS 4 Arm64 Test @@ -2343,6 +2365,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging photonos-5: name: Photon OS 5 Test @@ -2363,6 +2386,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging photonos-5-arm64: name: Photon OS 5 Arm64 Test @@ -2383,6 +2407,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging ubuntu-2004: name: Ubuntu 20.04 Test @@ -2403,6 +2428,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging ubuntu-2004-arm64: name: Ubuntu 20.04 Arm64 Test @@ -2423,6 +2449,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging ubuntu-2204: name: Ubuntu 22.04 Test @@ -2443,6 +2470,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging ubuntu-2204-arm64: name: Ubuntu 22.04 Arm64 Test @@ -2463,6 +2491,7 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13 skip-code-coverage: true skip-junit-reports: true + workflow-slug: staging build-src-repo: name: Build Repository diff --git a/.github/workflows/templates/nightly.yml.jinja b/.github/workflows/templates/nightly.yml.jinja index 67b07ad5ffa..e4f6bb8439e 100644 --- a/.github/workflows/templates/nightly.yml.jinja +++ b/.github/workflows/templates/nightly.yml.jinja @@ -28,7 +28,7 @@ on: description: Skip running the Salt packages test suite. schedule: # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onschedule - - cron: '0 1 * * *' # Every day at 1AM + - cron: '0 0 * * *' # Every day at 0AM <%- endblock on %> diff --git a/.github/workflows/templates/test-salt.yml.jinja b/.github/workflows/templates/test-salt.yml.jinja index 63834ee10dc..c135207601f 100644 --- a/.github/workflows/templates/test-salt.yml.jinja +++ b/.github/workflows/templates/test-salt.yml.jinja @@ -21,6 +21,7 @@ cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|<{ python_version }> skip-code-coverage: <{ skip_test_coverage_check }> skip-junit-reports: <{ skip_junit_reports_check }> + workflow-slug: <{ workflow_slug }> <%- endfor %> @@ -47,6 +48,7 @@ cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|<{ python_version }> skip-code-coverage: <{ skip_test_coverage_check }> skip-junit-reports: <{ skip_junit_reports_check }> + workflow-slug: <{ workflow_slug }> <%- endfor %> @@ -73,5 +75,6 @@ cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|<{ python_version }> skip-code-coverage: <{ skip_test_coverage_check }> skip-junit-reports: <{ skip_junit_reports_check }> + workflow-slug: <{ workflow_slug }> <%- endfor %> diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index a0dac7bdeef..a7fc8526bf2 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -56,6 +56,11 @@ on: type: boolean description: Skip Publishing JUnit Reports default: false + workflow-slug: + required: false + type: string + description: Which workflow is running. + default: ci env: COLUMNS: 190 @@ -85,7 +90,7 @@ jobs: - name: Generate Test Matrix id: generate-matrix run: | - tools ci matrix ${{ inputs.distro-slug }} + tools ci matrix --workflow=${{ inputs.workflow-slug }} ${{ inputs.distro-slug }} test: name: Test diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index cb9b1366dc3..79e1ec4a7fa 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -56,6 +56,11 @@ on: type: boolean description: Skip Publishing JUnit Reports default: false + workflow-slug: + required: false + type: string + description: Which workflow is running. + default: ci env: COLUMNS: 190 @@ -90,7 +95,7 @@ jobs: - name: Generate Test Matrix id: generate-matrix run: | - tools ci matrix ${{ fromJSON(inputs.testrun)['type'] == 'full' && '--full ' || '' }}${{ inputs.distro-slug }} + tools ci matrix --workflow=${{ inputs.workflow-slug }} ${{ fromJSON(inputs.testrun)['type'] == 'full' && '--full ' || '' }}${{ inputs.distro-slug }} test: name: Test diff --git a/tools/ci.py b/tools/ci.py index c026166ba7a..e09659bf513 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -621,9 +621,12 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path): "full": { "help": "Full test run", }, + "workflow": { + "help": "Which workflow is running", + }, }, ) -def matrix(ctx: Context, distro_slug: str, full: bool = False): +def matrix(ctx: Context, distro_slug: str, full: bool = False, workflow: str = "ci"): """ Generate the test matrix. """ @@ -634,6 +637,11 @@ def matrix(ctx: Context, distro_slug: str, full: bool = False): "scenarios": 1, "unit": 2, } + # On nightly and scheduled builds we don't want splits at all + if workflow.lower() in ("nightly", "scheduled"): + ctx.info(f"Clearning splits definition since workflow is '{workflow}'") + _splits.clear() + for transport in ("zeromq", "tcp"): if transport == "tcp": if distro_slug not in ( From 0d5a6bf7f7c03b160e344e1eca8ecb7ffb20ad56 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 28 Oct 2023 09:38:42 +0100 Subject: [PATCH 09/26] Switch to `tools.utils.gh.get_github_token` Signed-off-by: Pedro Algarvio --- tools/ci.py | 6 ++++-- tools/utils/__init__.py | 8 ++++++-- tools/utils/gh.py | 19 ++++++++++++++----- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/tools/ci.py b/tools/ci.py index e09659bf513..ddcf66c15b4 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any from ptscripts import Context, command_group import tools.utils +import tools.utils.gh if sys.version_info < (3, 11): from typing_extensions import NotRequired, TypedDict @@ -914,8 +915,9 @@ def _get_pr_test_labels_from_api( headers = { "Accept": "application/vnd.github+json", } - if "GITHUB_TOKEN" in os.environ: - headers["Authorization"] = f"Bearer {os.environ['GITHUB_TOKEN']}" + github_token = tools.utils.gh.get_github_token(ctx) + if github_token is not None: + headers["Authorization"] = f"Bearer {github_token}" web.headers.update(headers) ret = web.get(f"https://api.github.com/repos/{repository}/pulls/{pr}") if ret.status_code != 200: diff --git a/tools/utils/__init__.py b/tools/utils/__init__.py index f89633a668d..7ecca29992c 100644 --- a/tools/utils/__init__.py +++ b/tools/utils/__init__.py @@ -134,13 +134,17 @@ def get_salt_releases(ctx: Context, repository: str) -> list[Version]: """ Return a list of salt versions """ + # Deferred import + import tools.utils.gh + versions = set() with ctx.web as web: headers = { "Accept": "application/vnd.github+json", } - if "GITHUB_TOKEN" in os.environ: - headers["Authorization"] = f"Bearer {os.environ['GITHUB_TOKEN']}" + github_token = tools.utils.gh.get_github_token(ctx) + if github_token is not None: + headers["Authorization"] = f"Bearer {github_token}" web.headers.update(headers) ret = web.get(f"https://api.github.com/repos/{repository}/tags") if ret.status_code != 200: diff --git a/tools/utils/gh.py b/tools/utils/gh.py index a0895b132e0..34008936e4a 100644 --- a/tools/utils/gh.py +++ b/tools/utils/gh.py @@ -218,11 +218,20 @@ def get_github_token(ctx: Context) -> str | None: Get the GITHUB_TOKEN to be able to authenticate to the API. """ github_token = os.environ.get("GITHUB_TOKEN") - if github_token is None: - gh = shutil.which("gh") - ret = ctx.run(gh, "auth", "token", check=False, capture=True) - if ret.returncode == 0: - github_token = ret.stdout.decode().strip() or None + if github_token is not None: + ctx.info("$GITHUB_TOKEN was found on the environ") + return github_token + + gh = shutil.which("gh") + if gh is None: + ctx.info("The 'gh' CLI tool is not available. Can't get a token using it.") + return github_token + + ret = ctx.run(gh, "auth", "token", check=False, capture=True) + if ret.returncode == 0: + ctx.info("Got the GitHub token from the 'gh' CLI tool") + return ret.stdout.decode().strip() or None + ctx.info("Failed to get the GitHub token from the 'gh' CLI tool") return github_token From f0c3a19037961387d671a59ca94ae7b6cc7a402d Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 30 Oct 2023 22:06:13 +0000 Subject: [PATCH 10/26] When getting salt releases, on failure, try the main repository Signed-off-by: Pedro Algarvio --- tools/pkg/repo/publish.py | 7 ++++++- tools/utils/__init__.py | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/pkg/repo/publish.py b/tools/pkg/repo/publish.py index 56155b94a59..3ad0ec9e428 100644 --- a/tools/pkg/repo/publish.py +++ b/tools/pkg/repo/publish.py @@ -467,7 +467,12 @@ def github( with open(github_output, "a", encoding="utf-8") as wfh: wfh.write(f"release-messsage-file={release_message_path.resolve()}\n") - releases = get_salt_releases(ctx, repository) + try: + releases = get_salt_releases(ctx, repository) + except SystemExit: + ctx.warn(f"Failed to get salt releases from repository '{repository}'") + releases = get_salt_releases(ctx, "saltstack/salt") + if Version(salt_version) >= releases[-1]: make_latest = True else: diff --git a/tools/utils/__init__.py b/tools/utils/__init__.py index 7ecca29992c..b5dda0ddcb3 100644 --- a/tools/utils/__init__.py +++ b/tools/utils/__init__.py @@ -137,6 +137,8 @@ def get_salt_releases(ctx: Context, repository: str) -> list[Version]: # Deferred import import tools.utils.gh + ctx.info(f"Collecting salt releases from repository '{repository}'") + versions = set() with ctx.web as web: headers = { From 94a74fc3f363675c76c278b0f37fbf6a19dd93ed Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 1 Nov 2023 18:41:38 +0000 Subject: [PATCH 11/26] The default timeout values are now passed as inputs Signed-off-by: Pedro Algarvio --- .github/workflows/ci.yml | 29 +++++++++++++++++++ .github/workflows/nightly.yml | 29 +++++++++++++++++++ .github/workflows/scheduled.yml | 29 +++++++++++++++++++ .github/workflows/staging.yml | 29 +++++++++++++++++++ .../workflows/templates/test-salt.yml.jinja | 8 +++++ .github/workflows/test-action-macos.yml | 7 ++++- .github/workflows/test-action.yml | 7 ++++- 7 files changed, 136 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92cb24cd349..a105e974f8a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1848,6 +1848,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 windows-2019: name: Windows 2019 Test @@ -1869,6 +1870,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 windows-2022: name: Windows 2022 Test @@ -1890,6 +1892,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 macos-12: name: macOS 12 Test @@ -1911,6 +1914,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 almalinux-8: name: Alma Linux 8 Test @@ -1932,6 +1936,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 almalinux-9: name: Alma Linux 9 Test @@ -1953,6 +1958,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 amazonlinux-2: name: Amazon Linux 2 Test @@ -1974,6 +1980,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 archlinux-lts: name: Arch Linux LTS Test @@ -1995,6 +2002,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 centos-7: name: CentOS 7 Test @@ -2016,6 +2024,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 centosstream-8: name: CentOS Stream 8 Test @@ -2037,6 +2046,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 centosstream-9: name: CentOS Stream 9 Test @@ -2058,6 +2068,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 debian-10: name: Debian 10 Test @@ -2079,6 +2090,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 debian-11: name: Debian 11 Test @@ -2100,6 +2112,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 debian-11-arm64: name: Debian 11 Arm64 Test @@ -2121,6 +2134,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 debian-12: name: Debian 12 Test @@ -2142,6 +2156,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 debian-12-arm64: name: Debian 12 Arm64 Test @@ -2163,6 +2178,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 fedora-37: name: Fedora 37 Test @@ -2184,6 +2200,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 fedora-38: name: Fedora 38 Test @@ -2205,6 +2222,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 opensuse-15: name: Opensuse 15 Test @@ -2226,6 +2244,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 photonos-3: name: Photon OS 3 Test @@ -2247,6 +2266,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 photonos-3-arm64: name: Photon OS 3 Arm64 Test @@ -2268,6 +2288,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 photonos-4: name: Photon OS 4 Test @@ -2289,6 +2310,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 photonos-4-arm64: name: Photon OS 4 Arm64 Test @@ -2310,6 +2332,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 photonos-5: name: Photon OS 5 Test @@ -2331,6 +2354,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 photonos-5-arm64: name: Photon OS 5 Arm64 Test @@ -2352,6 +2376,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 ubuntu-2004: name: Ubuntu 20.04 Test @@ -2373,6 +2398,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 ubuntu-2004-arm64: name: Ubuntu 20.04 Arm64 Test @@ -2394,6 +2420,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 ubuntu-2204: name: Ubuntu 22.04 Test @@ -2415,6 +2442,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 ubuntu-2204-arm64: name: Ubuntu 22.04 Arm64 Test @@ -2436,6 +2464,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci + default-timeout: 120 combine-all-code-coverage: name: Combine Code Coverage diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e15743ed459..9224c915223 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1909,6 +1909,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 windows-2019: name: Windows 2019 Test @@ -1930,6 +1931,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 windows-2022: name: Windows 2022 Test @@ -1951,6 +1953,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 macos-12: name: macOS 12 Test @@ -1972,6 +1975,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 almalinux-8: name: Alma Linux 8 Test @@ -1993,6 +1997,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 almalinux-9: name: Alma Linux 9 Test @@ -2014,6 +2019,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 amazonlinux-2: name: Amazon Linux 2 Test @@ -2035,6 +2041,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 archlinux-lts: name: Arch Linux LTS Test @@ -2056,6 +2063,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 centos-7: name: CentOS 7 Test @@ -2077,6 +2085,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 centosstream-8: name: CentOS Stream 8 Test @@ -2098,6 +2107,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 centosstream-9: name: CentOS Stream 9 Test @@ -2119,6 +2129,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 debian-10: name: Debian 10 Test @@ -2140,6 +2151,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 debian-11: name: Debian 11 Test @@ -2161,6 +2173,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 debian-11-arm64: name: Debian 11 Arm64 Test @@ -2182,6 +2195,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 debian-12: name: Debian 12 Test @@ -2203,6 +2217,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 debian-12-arm64: name: Debian 12 Arm64 Test @@ -2224,6 +2239,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 fedora-37: name: Fedora 37 Test @@ -2245,6 +2261,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 fedora-38: name: Fedora 38 Test @@ -2266,6 +2283,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 opensuse-15: name: Opensuse 15 Test @@ -2287,6 +2305,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 photonos-3: name: Photon OS 3 Test @@ -2308,6 +2327,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 photonos-3-arm64: name: Photon OS 3 Arm64 Test @@ -2329,6 +2349,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 photonos-4: name: Photon OS 4 Test @@ -2350,6 +2371,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 photonos-4-arm64: name: Photon OS 4 Arm64 Test @@ -2371,6 +2393,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 photonos-5: name: Photon OS 5 Test @@ -2392,6 +2415,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 photonos-5-arm64: name: Photon OS 5 Arm64 Test @@ -2413,6 +2437,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 ubuntu-2004: name: Ubuntu 20.04 Test @@ -2434,6 +2459,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 ubuntu-2004-arm64: name: Ubuntu 20.04 Arm64 Test @@ -2455,6 +2481,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 ubuntu-2204: name: Ubuntu 22.04 Test @@ -2476,6 +2503,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 ubuntu-2204-arm64: name: Ubuntu 22.04 Arm64 Test @@ -2497,6 +2525,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly + default-timeout: 300 combine-all-code-coverage: name: Combine Code Coverage diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index d8a80b2790b..287dd7fa7a8 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -1882,6 +1882,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 windows-2019: name: Windows 2019 Test @@ -1903,6 +1904,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 windows-2022: name: Windows 2022 Test @@ -1924,6 +1926,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 macos-12: name: macOS 12 Test @@ -1945,6 +1948,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 almalinux-8: name: Alma Linux 8 Test @@ -1966,6 +1970,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 almalinux-9: name: Alma Linux 9 Test @@ -1987,6 +1992,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 amazonlinux-2: name: Amazon Linux 2 Test @@ -2008,6 +2014,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 archlinux-lts: name: Arch Linux LTS Test @@ -2029,6 +2036,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 centos-7: name: CentOS 7 Test @@ -2050,6 +2058,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 centosstream-8: name: CentOS Stream 8 Test @@ -2071,6 +2080,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 centosstream-9: name: CentOS Stream 9 Test @@ -2092,6 +2102,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 debian-10: name: Debian 10 Test @@ -2113,6 +2124,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 debian-11: name: Debian 11 Test @@ -2134,6 +2146,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 debian-11-arm64: name: Debian 11 Arm64 Test @@ -2155,6 +2168,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 debian-12: name: Debian 12 Test @@ -2176,6 +2190,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 debian-12-arm64: name: Debian 12 Arm64 Test @@ -2197,6 +2212,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 fedora-37: name: Fedora 37 Test @@ -2218,6 +2234,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 fedora-38: name: Fedora 38 Test @@ -2239,6 +2256,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 opensuse-15: name: Opensuse 15 Test @@ -2260,6 +2278,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 photonos-3: name: Photon OS 3 Test @@ -2281,6 +2300,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 photonos-3-arm64: name: Photon OS 3 Arm64 Test @@ -2302,6 +2322,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 photonos-4: name: Photon OS 4 Test @@ -2323,6 +2344,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 photonos-4-arm64: name: Photon OS 4 Arm64 Test @@ -2344,6 +2366,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 photonos-5: name: Photon OS 5 Test @@ -2365,6 +2388,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 photonos-5-arm64: name: Photon OS 5 Arm64 Test @@ -2386,6 +2410,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 ubuntu-2004: name: Ubuntu 20.04 Test @@ -2407,6 +2432,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 ubuntu-2004-arm64: name: Ubuntu 20.04 Arm64 Test @@ -2428,6 +2454,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 ubuntu-2204: name: Ubuntu 22.04 Test @@ -2449,6 +2476,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 ubuntu-2204-arm64: name: Ubuntu 22.04 Arm64 Test @@ -2470,6 +2498,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled + default-timeout: 300 combine-all-code-coverage: name: Combine Code Coverage diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 4962220fe5a..afdae3bd2f1 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -1904,6 +1904,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 windows-2019: name: Windows 2019 Test @@ -1925,6 +1926,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 windows-2022: name: Windows 2022 Test @@ -1946,6 +1948,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 macos-12: name: macOS 12 Test @@ -1967,6 +1970,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 almalinux-8: name: Alma Linux 8 Test @@ -1988,6 +1992,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 almalinux-9: name: Alma Linux 9 Test @@ -2009,6 +2014,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 amazonlinux-2: name: Amazon Linux 2 Test @@ -2030,6 +2036,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 archlinux-lts: name: Arch Linux LTS Test @@ -2051,6 +2058,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 centos-7: name: CentOS 7 Test @@ -2072,6 +2080,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 centosstream-8: name: CentOS Stream 8 Test @@ -2093,6 +2102,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 centosstream-9: name: CentOS Stream 9 Test @@ -2114,6 +2124,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 debian-10: name: Debian 10 Test @@ -2135,6 +2146,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 debian-11: name: Debian 11 Test @@ -2156,6 +2168,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 debian-11-arm64: name: Debian 11 Arm64 Test @@ -2177,6 +2190,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 debian-12: name: Debian 12 Test @@ -2198,6 +2212,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 debian-12-arm64: name: Debian 12 Arm64 Test @@ -2219,6 +2234,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 fedora-37: name: Fedora 37 Test @@ -2240,6 +2256,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 fedora-38: name: Fedora 38 Test @@ -2261,6 +2278,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 opensuse-15: name: Opensuse 15 Test @@ -2282,6 +2300,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 photonos-3: name: Photon OS 3 Test @@ -2303,6 +2322,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 photonos-3-arm64: name: Photon OS 3 Arm64 Test @@ -2324,6 +2344,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 photonos-4: name: Photon OS 4 Test @@ -2345,6 +2366,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 photonos-4-arm64: name: Photon OS 4 Arm64 Test @@ -2366,6 +2388,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 photonos-5: name: Photon OS 5 Test @@ -2387,6 +2410,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 photonos-5-arm64: name: Photon OS 5 Arm64 Test @@ -2408,6 +2432,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 ubuntu-2004: name: Ubuntu 20.04 Test @@ -2429,6 +2454,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 ubuntu-2004-arm64: name: Ubuntu 20.04 Arm64 Test @@ -2450,6 +2476,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 ubuntu-2204: name: Ubuntu 22.04 Test @@ -2471,6 +2498,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 ubuntu-2204-arm64: name: Ubuntu 22.04 Arm64 Test @@ -2492,6 +2520,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging + default-timeout: 120 build-src-repo: name: Build Repository diff --git a/.github/workflows/templates/test-salt.yml.jinja b/.github/workflows/templates/test-salt.yml.jinja index c135207601f..ce00a6efd1f 100644 --- a/.github/workflows/templates/test-salt.yml.jinja +++ b/.github/workflows/templates/test-salt.yml.jinja @@ -1,3 +1,8 @@ +<%- if workflow_slug in ("nightly", "scheduled") %> + <%- set timeout_value = 300 %> +<%- else %> + <%- set timeout_value = 120 %> +<%- endif %> <%- for slug, display_name, arch in test_salt_listing["windows"] %> @@ -22,6 +27,7 @@ skip-code-coverage: <{ skip_test_coverage_check }> skip-junit-reports: <{ skip_junit_reports_check }> workflow-slug: <{ workflow_slug }> + default-timeout: <{ timeout_value }> <%- endfor %> @@ -49,6 +55,7 @@ skip-code-coverage: <{ skip_test_coverage_check }> skip-junit-reports: <{ skip_junit_reports_check }> workflow-slug: <{ workflow_slug }> + default-timeout: <{ timeout_value }> <%- endfor %> @@ -76,5 +83,6 @@ skip-code-coverage: <{ skip_test_coverage_check }> skip-junit-reports: <{ skip_junit_reports_check }> workflow-slug: <{ workflow_slug }> + default-timeout: <{ timeout_value }> <%- endfor %> diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index a7fc8526bf2..ec84dec71c2 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -61,6 +61,11 @@ on: type: string description: Which workflow is running. default: ci + default-timeout: + required: false + type: number + description: Timeout, in minutes, for the test job(Default 300, 5 hours). + default: 300 env: COLUMNS: 190 @@ -97,7 +102,7 @@ jobs: runs-on: ${{ inputs.distro-slug }} # Full test runs. Each chunk should never take more than 2 hours. # Partial test runs(no chunk parallelization), 5 Hours - timeout-minutes: ${{ fromJSON(inputs.testrun)['type'] == 'full' && 120 || 300 }} + timeout-minutes: ${{ fromJSON(inputs.testrun)['type'] == 'full' && inputs.default-timeout || 300 }} needs: - generate-matrix strategy: diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index 79e1ec4a7fa..0f9fdf2b1e2 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -61,6 +61,11 @@ on: type: string description: Which workflow is running. default: ci + default-timeout: + required: false + type: number + description: Timeout, in minutes, for the test job(Default 300, 5 hours). + default: 300 env: COLUMNS: 190 @@ -105,7 +110,7 @@ jobs: - bastion # Full test runs. Each chunk should never take more than 2 hours. # Partial test runs(no chunk parallelization), 5 Hours - timeout-minutes: ${{ fromJSON(inputs.testrun)['type'] == 'full' && 120 || 300 }} + timeout-minutes: ${{ fromJSON(inputs.testrun)['type'] == 'full' && inputs.default-timeout || 300 }} needs: - generate-matrix strategy: From 42a2149621a2eedb631828d0214c3c3cb1486197 Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Tue, 31 Oct 2023 14:41:15 -0600 Subject: [PATCH 12/26] Ported tesT_listdiff.py from unittest to pytest and additional code-coverage tests --- tests/pytests/unit/utils/test_listdiffer.py | 186 ++++++++++++++++++++ tests/unit/utils/test_listdiffer.py | 109 ------------ 2 files changed, 186 insertions(+), 109 deletions(-) create mode 100644 tests/pytests/unit/utils/test_listdiffer.py delete mode 100644 tests/unit/utils/test_listdiffer.py diff --git a/tests/pytests/unit/utils/test_listdiffer.py b/tests/pytests/unit/utils/test_listdiffer.py new file mode 100644 index 00000000000..6ebf07103e1 --- /dev/null +++ b/tests/pytests/unit/utils/test_listdiffer.py @@ -0,0 +1,186 @@ +import pytest + +from salt.utils import dictdiffer +from salt.utils.listdiffer import list_diff + + +@pytest.fixture +def get_NONE_value(): + return dictdiffer.RecursiveDictDiffer.NONE_VALUE + + +@pytest.fixture +def get_old_list(): + return [ + {"key": 1, "value": "foo1", "int_value": 101}, + {"key": 2, "value": "foo2", "int_value": 102}, + {"key": 3, "value": "foo3", "int_value": 103}, + ] + + +@pytest.fixture +def get_new_list(): + return [ + {"key": 1, "value": "foo1", "int_value": 101}, + {"key": 2, "value": "foo2", "int_value": 112}, + {"key": 5, "value": "foo5", "int_value": 105}, + ] + + +@pytest.fixture +def test_list_diff(get_old_list, get_new_list): + test_list_diff = list_diff(get_old_list, get_new_list, key="key") + return test_list_diff + + +def test_added(test_list_diff): + assert len(test_list_diff.added) == 1 + assert test_list_diff.added[0] == {"key": 5, "value": "foo5", "int_value": 105} + + +def test_removed(test_list_diff): + assert len(test_list_diff.removed) == 1 + assert test_list_diff.removed[0] == {"key": 3, "value": "foo3", "int_value": 103} + + +def test_diffs(test_list_diff, get_NONE_value): + assert len(test_list_diff.diffs) == 3 + assert test_list_diff.diffs[0] == {2: {"int_value": {"new": 112, "old": 102}}} + + # Added items + assert test_list_diff.diffs[1] == { + 5: { + "int_value": {"new": 105, "old": get_NONE_value}, + "key": {"new": 5, "old": get_NONE_value}, + "value": {"new": "foo5", "old": get_NONE_value}, + } + } + + # Removed items + assert test_list_diff.diffs[2] == { + 3: { + "int_value": {"new": get_NONE_value, "old": 103}, + "key": {"new": get_NONE_value, "old": 3}, + "value": {"new": get_NONE_value, "old": "foo3"}, + } + } + + +def test_new_values(test_list_diff): + assert len(test_list_diff.new_values) == 2 + assert test_list_diff.new_values[0] == {"key": 2, "int_value": 112} + assert test_list_diff.new_values[1] == {"key": 5, "value": "foo5", "int_value": 105} + + +def test_old_values(test_list_diff): + assert len(test_list_diff.old_values) == 2 + assert test_list_diff.old_values[0] == {"key": 2, "int_value": 102} + assert test_list_diff.old_values[1] == {"key": 3, "value": "foo3", "int_value": 103} + + +def test_changed_all(test_list_diff): + assert test_list_diff.changed(selection="all") == [ + "key.2.int_value", + "key.5.int_value", + "key.5.value", + "key.3.int_value", + "key.3.value", + ] + + +def test_changed_intersect(test_list_diff): + assert test_list_diff.changed(selection="intersect") == ["key.2.int_value"] + + +def test_changes_str(test_list_diff): + expected = """\tidentified by key 2: +\tint_value from 102 to 112 +\tidentified by key 3: +\twill be removed +\tidentified by key 5: +\twill be added +""" + assert test_list_diff.changes_str == expected + + +def test_intersect(test_list_diff): + expected = [ + { + "key": 1, + "old": {"key": 1, "value": "foo1", "int_value": 101}, + "new": {"key": 1, "value": "foo1", "int_value": 101}, + }, + { + "key": 2, + "old": {"key": 2, "value": "foo2", "int_value": 102}, + "new": {"key": 2, "value": "foo2", "int_value": 112}, + }, + ] + test_isect = test_list_diff.intersect + assert test_isect == expected + + +def test_remove_diff_intersect(test_list_diff): + expected = [ + { + "key": 1, + "old": {"key": 1, "int_value": 101}, + "new": {"key": 1, "int_value": 101}, + }, + { + "key": 2, + "old": {"key": 2, "int_value": 102}, + "new": {"key": 2, "int_value": 112}, + }, + ] + + test_list_diff.remove_diff(diff_key="value") + test_isect = test_list_diff.intersect + assert test_isect == expected + + +def test_remove_diff_removed(test_list_diff): + expected = [ + { + "key": 1, + "old": {"key": 1, "value": "foo1", "int_value": 101}, + "new": {"key": 1, "value": "foo1", "int_value": 101}, + }, + { + "key": 2, + "old": {"key": 2, "value": "foo2", "int_value": 102}, + "new": {"key": 2, "value": "foo2", "int_value": 112}, + }, + ] + test_list_diff.remove_diff(diff_key="value", diff_list="removed") + test_isect = test_list_diff.intersect + assert test_isect == expected + + +def test_changes_str2(test_list_diff): + expected = """ key=2 (updated): + int_value from 102 to 112 + key=3 (removed) + key=5 (added): {'key': 5, 'value': 'foo5', 'int_value': 105}""" + test_changes = test_list_diff.changes_str2 + assert test_changes == expected + + +def test_current_list(test_list_diff): + expected = [ + {"key": 1, "value": "foo1", "int_value": 101}, + {"key": 2, "value": "foo2", "int_value": 102}, + {"key": 3, "value": "foo3", "int_value": 103}, + ] + test_curr_list = test_list_diff.current_list + assert test_curr_list == expected + + +def test_new_list(test_list_diff): + expected = [ + {"key": 1, "value": "foo1", "int_value": 101}, + {"key": 2, "value": "foo2", "int_value": 112}, + {"key": 5, "value": "foo5", "int_value": 105}, + ] + test_new_list = test_list_diff.new_list + assert test_new_list == expected diff --git a/tests/unit/utils/test_listdiffer.py b/tests/unit/utils/test_listdiffer.py deleted file mode 100644 index cd1922a5f6e..00000000000 --- a/tests/unit/utils/test_listdiffer.py +++ /dev/null @@ -1,109 +0,0 @@ -from salt.utils import dictdiffer -from salt.utils.listdiffer import list_diff -from tests.support.unit import TestCase - -NONE = dictdiffer.RecursiveDictDiffer.NONE_VALUE - - -class ListDictDifferTestCase(TestCase): - def setUp(self): - old_list = [ - {"key": 1, "value": "foo1", "int_value": 101}, - {"key": 2, "value": "foo2", "int_value": 102}, - {"key": 3, "value": "foo3", "int_value": 103}, - ] - new_list = [ - {"key": 1, "value": "foo1", "int_value": 101}, - {"key": 2, "value": "foo2", "int_value": 112}, - {"key": 5, "value": "foo5", "int_value": 105}, - ] - self.list_diff = list_diff(old_list, new_list, key="key") - - def tearDown(self): - for attrname in ("list_diff",): - try: - delattr(self, attrname) - except AttributeError: - continue - - def test_added(self): - self.assertEqual(len(self.list_diff.added), 1) - self.assertDictEqual( - self.list_diff.added[0], {"key": 5, "value": "foo5", "int_value": 105} - ) - - def test_removed(self): - self.assertEqual(len(self.list_diff.removed), 1) - self.assertDictEqual( - self.list_diff.removed[0], {"key": 3, "value": "foo3", "int_value": 103} - ) - - def test_diffs(self): - self.assertEqual(len(self.list_diff.diffs), 3) - self.assertDictEqual( - self.list_diff.diffs[0], {2: {"int_value": {"new": 112, "old": 102}}} - ) - self.assertDictEqual( - self.list_diff.diffs[1], - # Added items - { - 5: { - "int_value": {"new": 105, "old": NONE}, - "key": {"new": 5, "old": NONE}, - "value": {"new": "foo5", "old": NONE}, - } - }, - ) - self.assertDictEqual( - self.list_diff.diffs[2], - # Removed items - { - 3: { - "int_value": {"new": NONE, "old": 103}, - "key": {"new": NONE, "old": 3}, - "value": {"new": NONE, "old": "foo3"}, - } - }, - ) - - def test_new_values(self): - self.assertEqual(len(self.list_diff.new_values), 2) - self.assertDictEqual(self.list_diff.new_values[0], {"key": 2, "int_value": 112}) - self.assertDictEqual( - self.list_diff.new_values[1], {"key": 5, "value": "foo5", "int_value": 105} - ) - - def test_old_values(self): - self.assertEqual(len(self.list_diff.old_values), 2) - self.assertDictEqual(self.list_diff.old_values[0], {"key": 2, "int_value": 102}) - self.assertDictEqual( - self.list_diff.old_values[1], {"key": 3, "value": "foo3", "int_value": 103} - ) - - def test_changed_all(self): - self.assertEqual( - self.list_diff.changed(selection="all"), - [ - "key.2.int_value", - "key.5.int_value", - "key.5.value", - "key.3.int_value", - "key.3.value", - ], - ) - - def test_changed_intersect(self): - self.assertEqual( - self.list_diff.changed(selection="intersect"), ["key.2.int_value"] - ) - - def test_changes_str(self): - self.assertEqual( - self.list_diff.changes_str, - "\tidentified by key 2:\n" - "\tint_value from 102 to 112\n" - "\tidentified by key 3:\n" - "\twill be removed\n" - "\tidentified by key 5:\n" - "\twill be added\n", - ) From 29a6202f96663b2bfb2ee264d9caf2be3c9380da Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Wed, 1 Nov 2023 11:22:14 -0600 Subject: [PATCH 13/26] Updatd test per reviewer comments --- tests/pytests/unit/utils/test_listdiffer.py | 102 +++++++++----------- 1 file changed, 48 insertions(+), 54 deletions(-) diff --git a/tests/pytests/unit/utils/test_listdiffer.py b/tests/pytests/unit/utils/test_listdiffer.py index 6ebf07103e1..dfc42a4fe9f 100644 --- a/tests/pytests/unit/utils/test_listdiffer.py +++ b/tests/pytests/unit/utils/test_listdiffer.py @@ -1,14 +1,9 @@ import pytest -from salt.utils import dictdiffer +from salt.utils.dictdiffer import RecursiveDictDiffer from salt.utils.listdiffer import list_diff -@pytest.fixture -def get_NONE_value(): - return dictdiffer.RecursiveDictDiffer.NONE_VALUE - - @pytest.fixture def get_old_list(): return [ @@ -28,58 +23,57 @@ def get_new_list(): @pytest.fixture -def test_list_diff(get_old_list, get_new_list): - test_list_diff = list_diff(get_old_list, get_new_list, key="key") - return test_list_diff +def get_list_diff(get_old_list, get_new_list): + return list_diff(get_old_list, get_new_list, key="key") -def test_added(test_list_diff): - assert len(test_list_diff.added) == 1 - assert test_list_diff.added[0] == {"key": 5, "value": "foo5", "int_value": 105} +def test_added(get_list_diff): + assert len(get_list_diff.added) == 1 + assert get_list_diff.added[0] == {"key": 5, "value": "foo5", "int_value": 105} -def test_removed(test_list_diff): - assert len(test_list_diff.removed) == 1 - assert test_list_diff.removed[0] == {"key": 3, "value": "foo3", "int_value": 103} +def test_removed(get_list_diff): + assert len(get_list_diff.removed) == 1 + assert get_list_diff.removed[0] == {"key": 3, "value": "foo3", "int_value": 103} -def test_diffs(test_list_diff, get_NONE_value): - assert len(test_list_diff.diffs) == 3 - assert test_list_diff.diffs[0] == {2: {"int_value": {"new": 112, "old": 102}}} +def test_diffs(get_list_diff): + assert len(get_list_diff.diffs) == 3 + assert get_list_diff.diffs[0] == {2: {"int_value": {"new": 112, "old": 102}}} # Added items - assert test_list_diff.diffs[1] == { + assert get_list_diff.diffs[1] == { 5: { - "int_value": {"new": 105, "old": get_NONE_value}, - "key": {"new": 5, "old": get_NONE_value}, - "value": {"new": "foo5", "old": get_NONE_value}, + "int_value": {"new": 105, "old": RecursiveDictDiffer.NONE_VALUE}, + "key": {"new": 5, "old": RecursiveDictDiffer.NONE_VALUE}, + "value": {"new": "foo5", "old": RecursiveDictDiffer.NONE_VALUE}, } } # Removed items - assert test_list_diff.diffs[2] == { + assert get_list_diff.diffs[2] == { 3: { - "int_value": {"new": get_NONE_value, "old": 103}, - "key": {"new": get_NONE_value, "old": 3}, - "value": {"new": get_NONE_value, "old": "foo3"}, + "int_value": {"new": RecursiveDictDiffer.NONE_VALUE, "old": 103}, + "key": {"new": RecursiveDictDiffer.NONE_VALUE, "old": 3}, + "value": {"new": RecursiveDictDiffer.NONE_VALUE, "old": "foo3"}, } } -def test_new_values(test_list_diff): - assert len(test_list_diff.new_values) == 2 - assert test_list_diff.new_values[0] == {"key": 2, "int_value": 112} - assert test_list_diff.new_values[1] == {"key": 5, "value": "foo5", "int_value": 105} +def test_new_values(get_list_diff): + assert len(get_list_diff.new_values) == 2 + assert get_list_diff.new_values[0] == {"key": 2, "int_value": 112} + assert get_list_diff.new_values[1] == {"key": 5, "value": "foo5", "int_value": 105} -def test_old_values(test_list_diff): - assert len(test_list_diff.old_values) == 2 - assert test_list_diff.old_values[0] == {"key": 2, "int_value": 102} - assert test_list_diff.old_values[1] == {"key": 3, "value": "foo3", "int_value": 103} +def test_old_values(get_list_diff): + assert len(get_list_diff.old_values) == 2 + assert get_list_diff.old_values[0] == {"key": 2, "int_value": 102} + assert get_list_diff.old_values[1] == {"key": 3, "value": "foo3", "int_value": 103} -def test_changed_all(test_list_diff): - assert test_list_diff.changed(selection="all") == [ +def test_changed_all(get_list_diff): + assert get_list_diff.changed(selection="all") == [ "key.2.int_value", "key.5.int_value", "key.5.value", @@ -88,11 +82,11 @@ def test_changed_all(test_list_diff): ] -def test_changed_intersect(test_list_diff): - assert test_list_diff.changed(selection="intersect") == ["key.2.int_value"] +def test_changed_intersect(get_list_diff): + assert get_list_diff.changed(selection="intersect") == ["key.2.int_value"] -def test_changes_str(test_list_diff): +def test_changes_str(get_list_diff): expected = """\tidentified by key 2: \tint_value from 102 to 112 \tidentified by key 3: @@ -100,10 +94,10 @@ def test_changes_str(test_list_diff): \tidentified by key 5: \twill be added """ - assert test_list_diff.changes_str == expected + assert get_list_diff.changes_str == expected -def test_intersect(test_list_diff): +def test_intersect(get_list_diff): expected = [ { "key": 1, @@ -116,11 +110,11 @@ def test_intersect(test_list_diff): "new": {"key": 2, "value": "foo2", "int_value": 112}, }, ] - test_isect = test_list_diff.intersect + test_isect = get_list_diff.intersect assert test_isect == expected -def test_remove_diff_intersect(test_list_diff): +def test_remove_diff_intersect(get_list_diff): expected = [ { "key": 1, @@ -134,12 +128,12 @@ def test_remove_diff_intersect(test_list_diff): }, ] - test_list_diff.remove_diff(diff_key="value") - test_isect = test_list_diff.intersect + get_list_diff.remove_diff(diff_key="value") + test_isect = get_list_diff.intersect assert test_isect == expected -def test_remove_diff_removed(test_list_diff): +def test_remove_diff_removed(get_list_diff): expected = [ { "key": 1, @@ -152,35 +146,35 @@ def test_remove_diff_removed(test_list_diff): "new": {"key": 2, "value": "foo2", "int_value": 112}, }, ] - test_list_diff.remove_diff(diff_key="value", diff_list="removed") - test_isect = test_list_diff.intersect + get_list_diff.remove_diff(diff_key="value", diff_list="removed") + test_isect = get_list_diff.intersect assert test_isect == expected -def test_changes_str2(test_list_diff): +def test_changes_str2(get_list_diff): expected = """ key=2 (updated): int_value from 102 to 112 key=3 (removed) key=5 (added): {'key': 5, 'value': 'foo5', 'int_value': 105}""" - test_changes = test_list_diff.changes_str2 + test_changes = get_list_diff.changes_str2 assert test_changes == expected -def test_current_list(test_list_diff): +def test_current_list(get_list_diff): expected = [ {"key": 1, "value": "foo1", "int_value": 101}, {"key": 2, "value": "foo2", "int_value": 102}, {"key": 3, "value": "foo3", "int_value": 103}, ] - test_curr_list = test_list_diff.current_list + test_curr_list = get_list_diff.current_list assert test_curr_list == expected -def test_new_list(test_list_diff): +def test_new_list(get_list_diff): expected = [ {"key": 1, "value": "foo1", "int_value": 101}, {"key": 2, "value": "foo2", "int_value": 112}, {"key": 5, "value": "foo5", "int_value": 105}, ] - test_new_list = test_list_diff.new_list + test_new_list = get_list_diff.new_list assert test_new_list == expected From 4fd5072dbd5dc941990de4543b37c9e2a7a9d5bc Mon Sep 17 00:00:00 2001 From: cmcmarrow Date: Thu, 19 Oct 2023 12:43:07 -0500 Subject: [PATCH 14/26] bumpy pygit2 --- requirements/static/ci/darwin.in | 2 +- requirements/static/ci/freebsd.in | 2 +- requirements/static/ci/linux.in | 4 +--- requirements/static/ci/py3.10/darwin.txt | 2 +- requirements/static/ci/py3.10/freebsd.txt | 2 +- requirements/static/ci/py3.10/lint.txt | 2 +- requirements/static/ci/py3.10/linux.txt | 2 +- requirements/static/ci/py3.10/windows.txt | 2 +- requirements/static/ci/py3.7/freebsd.txt | 4 +++- requirements/static/ci/py3.7/lint.txt | 6 +++++- requirements/static/ci/py3.7/linux.txt | 4 +++- requirements/static/ci/py3.7/windows.txt | 2 +- requirements/static/ci/py3.8/freebsd.txt | 2 +- requirements/static/ci/py3.8/lint.txt | 2 +- requirements/static/ci/py3.8/linux.txt | 2 +- requirements/static/ci/py3.8/windows.txt | 2 +- requirements/static/ci/py3.9/darwin.txt | 2 +- requirements/static/ci/py3.9/freebsd.txt | 2 +- requirements/static/ci/py3.9/lint.txt | 6 +----- requirements/static/ci/py3.9/linux.txt | 4 +--- requirements/static/ci/py3.9/windows.txt | 2 +- requirements/static/ci/windows.in | 2 +- 22 files changed, 30 insertions(+), 30 deletions(-) diff --git a/requirements/static/ci/darwin.in b/requirements/static/ci/darwin.in index 49b95223e8b..bd95f1f41b7 100644 --- a/requirements/static/ci/darwin.in +++ b/requirements/static/ci/darwin.in @@ -5,7 +5,7 @@ --constraint=../pkg/py{py_version}/{platform}.txt yamlordereddictloader -pygit2>=1.2.0 +pygit2>=1.10.1 yamllint mercurial hglib diff --git a/requirements/static/ci/freebsd.in b/requirements/static/ci/freebsd.in index b628d6514a2..1fad585d0c9 100644 --- a/requirements/static/ci/freebsd.in +++ b/requirements/static/ci/freebsd.in @@ -1,7 +1,7 @@ # FreeBSD static CI requirements --constraint=../pkg/py{py_version}/{platform}.txt -pygit2==1.8.0 +pygit2>=1.10.1 yamllint mercurial hglib diff --git a/requirements/static/ci/linux.in b/requirements/static/ci/linux.in index 34c2c054a27..203d66ce71f 100644 --- a/requirements/static/ci/linux.in +++ b/requirements/static/ci/linux.in @@ -2,9 +2,7 @@ --constraint=../pkg/py{py_version}/{platform}.txt pyiface -pygit2<1.1.0; python_version <= '3.8' -pygit2>=1.4.0; python_version > '3.8' -pygit2==1.9.1; python_version >= '3.10' +pygit2>=1.10.1 pymysql>=1.0.2 ansible>=4.4.0; python_version < '3.9' ansible>=7.0.0; python_version >= '3.9' diff --git a/requirements/static/ci/py3.10/darwin.txt b/requirements/static/ci/py3.10/darwin.txt index ab71f7987bf..3da6a738ab8 100644 --- a/requirements/static/ci/py3.10/darwin.txt +++ b/requirements/static/ci/py3.10/darwin.txt @@ -707,7 +707,7 @@ pycryptodomex==3.9.8 # via # -c requirements/static/ci/../pkg/py3.10/darwin.txt # -r requirements/crypto.txt -pygit2==1.9.1 +pygit2==1.13.1 # via -r requirements/static/ci/darwin.in pyjwt==2.4.0 # via adal diff --git a/requirements/static/ci/py3.10/freebsd.txt b/requirements/static/ci/py3.10/freebsd.txt index ab3910913c4..6d379cac9c5 100644 --- a/requirements/static/ci/py3.10/freebsd.txt +++ b/requirements/static/ci/py3.10/freebsd.txt @@ -698,7 +698,7 @@ pycryptodomex==3.9.8 # via # -c requirements/static/ci/../pkg/py3.10/freebsd.txt # -r requirements/crypto.txt -pygit2==1.8.0 +pygit2==1.13.1 # via -r requirements/static/ci/freebsd.in pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in diff --git a/requirements/static/ci/py3.10/lint.txt b/requirements/static/ci/py3.10/lint.txt index a0eb57db663..ec68133ad30 100644 --- a/requirements/static/ci/py3.10/lint.txt +++ b/requirements/static/ci/py3.10/lint.txt @@ -993,7 +993,7 @@ pycryptodomex==3.9.8 # -c requirements/static/ci/../pkg/py3.10/linux.txt # -c requirements/static/ci/py3.10/linux.txt # -r requirements/crypto.txt -pygit2==1.9.1 ; python_version >= "3.10" +pygit2==1.13.1 # via # -c requirements/static/ci/py3.10/linux.txt # -r requirements/static/ci/linux.in diff --git a/requirements/static/ci/py3.10/linux.txt b/requirements/static/ci/py3.10/linux.txt index 2a267e08d2f..b3c8aa5556e 100644 --- a/requirements/static/ci/py3.10/linux.txt +++ b/requirements/static/ci/py3.10/linux.txt @@ -711,7 +711,7 @@ pycryptodomex==3.9.8 # via # -c requirements/static/ci/../pkg/py3.10/linux.txt # -r requirements/crypto.txt -pygit2==1.9.1 ; python_version >= "3.10" +pygit2==1.13.1 # via -r requirements/static/ci/linux.in pyiface==0.0.11 # via -r requirements/static/ci/linux.in diff --git a/requirements/static/ci/py3.10/windows.txt b/requirements/static/ci/py3.10/windows.txt index aa68c670856..5646b9a2930 100644 --- a/requirements/static/ci/py3.10/windows.txt +++ b/requirements/static/ci/py3.10/windows.txt @@ -278,7 +278,7 @@ pycryptodomex==3.10.1 # via # -c requirements/static/ci/../pkg/py3.10/windows.txt # -r requirements/crypto.txt -pygit2==1.9.1 +pygit2==1.13.1 # via -r requirements/static/ci/windows.in pymssql==2.2.7 # via diff --git a/requirements/static/ci/py3.7/freebsd.txt b/requirements/static/ci/py3.7/freebsd.txt index 281969ebf98..838981a8aa1 100644 --- a/requirements/static/ci/py3.7/freebsd.txt +++ b/requirements/static/ci/py3.7/freebsd.txt @@ -346,6 +346,8 @@ botocore==1.24.46 # boto3 # moto # s3transfer +cached-property==1.5.2 + # via pygit2 cachetools==3.1.0 # via google-auth cassandra-driver==3.24.0 @@ -735,7 +737,7 @@ pycryptodomex==3.9.8 # -r requirements/crypto.txt pyeapi==0.8.3 # via napalm -pygit2==1.8.0 +pygit2==1.10.1 # via -r requirements/static/ci/freebsd.in pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in diff --git a/requirements/static/ci/py3.7/lint.txt b/requirements/static/ci/py3.7/lint.txt index 2e9f5f55d0b..63ac0a28c01 100644 --- a/requirements/static/ci/py3.7/lint.txt +++ b/requirements/static/ci/py3.7/lint.txt @@ -543,6 +543,10 @@ botocore==1.24.46 # boto3 # moto # s3transfer +cached-property==1.5.2 + # via + # -c requirements/static/ci/py3.7/linux.txt + # pygit2 cachetools==4.2.2 # via # -c requirements/static/ci/py3.7/linux.txt @@ -1044,7 +1048,7 @@ pyeapi==0.8.3 # via # -c requirements/static/ci/py3.7/linux.txt # napalm -pygit2==1.0.3 ; python_version <= "3.8" +pygit2==1.10.1 # via # -c requirements/static/ci/py3.7/linux.txt # -r requirements/static/ci/linux.in diff --git a/requirements/static/ci/py3.7/linux.txt b/requirements/static/ci/py3.7/linux.txt index c66aae87b32..c077e6a3e72 100644 --- a/requirements/static/ci/py3.7/linux.txt +++ b/requirements/static/ci/py3.7/linux.txt @@ -359,6 +359,8 @@ botocore==1.24.46 # boto3 # moto # s3transfer +cached-property==1.5.2 + # via pygit2 cachetools==4.2.2 # via # google-auth @@ -750,7 +752,7 @@ pycryptodomex==3.9.8 # -r requirements/crypto.txt pyeapi==0.8.3 # via napalm -pygit2==1.0.3 ; python_version <= "3.8" +pygit2==1.10.1 # via -r requirements/static/ci/linux.in pyiface==0.0.11 # via -r requirements/static/ci/linux.in diff --git a/requirements/static/ci/py3.7/windows.txt b/requirements/static/ci/py3.7/windows.txt index 358a269c22d..dfa1ed14327 100644 --- a/requirements/static/ci/py3.7/windows.txt +++ b/requirements/static/ci/py3.7/windows.txt @@ -291,7 +291,7 @@ pycryptodomex==3.10.1 # via # -c requirements/static/ci/../pkg/py3.7/windows.txt # -r requirements/crypto.txt -pygit2==1.9.1 +pygit2==1.10.1 # via -r requirements/static/ci/windows.in pymssql==2.2.1 # via diff --git a/requirements/static/ci/py3.8/freebsd.txt b/requirements/static/ci/py3.8/freebsd.txt index ed7fe4bf658..7a136305dd4 100644 --- a/requirements/static/ci/py3.8/freebsd.txt +++ b/requirements/static/ci/py3.8/freebsd.txt @@ -725,7 +725,7 @@ pycryptodomex==3.9.8 # -r requirements/crypto.txt pyeapi==0.8.3 # via napalm -pygit2==1.8.0 +pygit2==1.13.1 # via -r requirements/static/ci/freebsd.in pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in diff --git a/requirements/static/ci/py3.8/lint.txt b/requirements/static/ci/py3.8/lint.txt index 3ff68e1d3a6..cd61ff37d59 100644 --- a/requirements/static/ci/py3.8/lint.txt +++ b/requirements/static/ci/py3.8/lint.txt @@ -1034,7 +1034,7 @@ pyeapi==0.8.3 # via # -c requirements/static/ci/py3.8/linux.txt # napalm -pygit2==1.0.3 ; python_version <= "3.8" +pygit2==1.13.1 # via # -c requirements/static/ci/py3.8/linux.txt # -r requirements/static/ci/linux.in diff --git a/requirements/static/ci/py3.8/linux.txt b/requirements/static/ci/py3.8/linux.txt index 5269b54d76f..1d1bf99db88 100644 --- a/requirements/static/ci/py3.8/linux.txt +++ b/requirements/static/ci/py3.8/linux.txt @@ -740,7 +740,7 @@ pycryptodomex==3.9.8 # -r requirements/crypto.txt pyeapi==0.8.3 # via napalm -pygit2==1.0.3 ; python_version <= "3.8" +pygit2==1.13.1 # via -r requirements/static/ci/linux.in pyiface==0.0.11 # via -r requirements/static/ci/linux.in diff --git a/requirements/static/ci/py3.8/windows.txt b/requirements/static/ci/py3.8/windows.txt index 12557cc1db2..acd53a81525 100644 --- a/requirements/static/ci/py3.8/windows.txt +++ b/requirements/static/ci/py3.8/windows.txt @@ -279,7 +279,7 @@ pycryptodomex==3.10.1 # via # -c requirements/static/ci/../pkg/py3.8/windows.txt # -r requirements/crypto.txt -pygit2==1.9.1 +pygit2==1.13.1 # via -r requirements/static/ci/windows.in pymssql==2.2.1 # via diff --git a/requirements/static/ci/py3.9/darwin.txt b/requirements/static/ci/py3.9/darwin.txt index 4d427fc1921..b2141a34c4e 100644 --- a/requirements/static/ci/py3.9/darwin.txt +++ b/requirements/static/ci/py3.9/darwin.txt @@ -736,7 +736,7 @@ pycryptodomex==3.9.8 # -r requirements/crypto.txt pyeapi==0.8.3 # via napalm -pygit2==1.9.1 +pygit2==1.13.1 # via -r requirements/static/ci/darwin.in pyjwt==2.4.0 # via adal diff --git a/requirements/static/ci/py3.9/freebsd.txt b/requirements/static/ci/py3.9/freebsd.txt index 1986be0a083..8e5260f63e8 100644 --- a/requirements/static/ci/py3.9/freebsd.txt +++ b/requirements/static/ci/py3.9/freebsd.txt @@ -727,7 +727,7 @@ pycryptodomex==3.9.8 # -r requirements/crypto.txt pyeapi==0.8.3 # via napalm -pygit2==1.8.0 +pygit2==1.13.1 # via -r requirements/static/ci/freebsd.in pyinotify==0.9.6 ; sys_platform != "win32" and sys_platform != "darwin" and platform_system != "openbsd" # via -r requirements/static/ci/common.in diff --git a/requirements/static/ci/py3.9/lint.txt b/requirements/static/ci/py3.9/lint.txt index 8f065ba5811..c9321d59f3f 100644 --- a/requirements/static/ci/py3.9/lint.txt +++ b/requirements/static/ci/py3.9/lint.txt @@ -535,10 +535,6 @@ botocore==1.24.46 # boto3 # moto # s3transfer -cached-property==1.5.2 - # via - # -c requirements/static/ci/py3.9/linux.txt - # pygit2 cachetools==4.2.2 # via # -c requirements/static/ci/py3.9/linux.txt @@ -1036,7 +1032,7 @@ pyeapi==0.8.3 # via # -c requirements/static/ci/py3.9/linux.txt # napalm -pygit2==1.5.0 ; python_version > "3.8" +pygit2==1.13.1 # via # -c requirements/static/ci/py3.9/linux.txt # -r requirements/static/ci/linux.in diff --git a/requirements/static/ci/py3.9/linux.txt b/requirements/static/ci/py3.9/linux.txt index 0c5f49494bf..b11e15185cc 100644 --- a/requirements/static/ci/py3.9/linux.txt +++ b/requirements/static/ci/py3.9/linux.txt @@ -355,8 +355,6 @@ botocore==1.24.46 # boto3 # moto # s3transfer -cached-property==1.5.2 - # via pygit2 cachetools==4.2.2 # via # google-auth @@ -742,7 +740,7 @@ pycryptodomex==3.9.8 # -r requirements/crypto.txt pyeapi==0.8.3 # via napalm -pygit2==1.5.0 ; python_version > "3.8" +pygit2==1.13.1 # via -r requirements/static/ci/linux.in pyiface==0.0.11 # via -r requirements/static/ci/linux.in diff --git a/requirements/static/ci/py3.9/windows.txt b/requirements/static/ci/py3.9/windows.txt index fbac0ca855d..0fdf07ef814 100644 --- a/requirements/static/ci/py3.9/windows.txt +++ b/requirements/static/ci/py3.9/windows.txt @@ -280,7 +280,7 @@ pycryptodomex==3.10.1 # via # -c requirements/static/ci/../pkg/py3.9/windows.txt # -r requirements/crypto.txt -pygit2==1.9.1 +pygit2==1.13.1 # via -r requirements/static/ci/windows.in pymssql==2.2.1 # via diff --git a/requirements/static/ci/windows.in b/requirements/static/ci/windows.in index f17f2b738a6..52afea9a71e 100644 --- a/requirements/static/ci/windows.in +++ b/requirements/static/ci/windows.in @@ -3,7 +3,7 @@ dmidecode patch -pygit2>=1.2.0 +pygit2>=1.10.1 sed pywinrm>=0.4.1 yamllint From 1abcf81601b27938c231f04305a97fe08e639112 Mon Sep 17 00:00:00 2001 From: Barney Sowood Date: Fri, 29 Sep 2023 17:48:22 +0100 Subject: [PATCH 15/26] Fix install of logrotate config for debian pkg Fixes the install of logrotate config for the debian pkg by moving file to pkg/common/logrotate/salt-common. File is installed via salt-common.install file, which can't rename files, only copy them to a directory, so we need to rename the file and put it in a subdir of pkg/common. Adds /etc/logrotate.d/salt-common to salt.common.conffiles to ensure that dpkg will not overwrite configs modified by users. Also updates RPM spec file for new location of logrotate config. Config needs to be /etc/logrotate.d/salt-common as that is what is used by 3005.x packages. --- pkg/common/{salt-common.logrotate => logrotate/salt-common} | 0 pkg/debian/salt-common.conffiles | 1 + pkg/debian/salt-common.dirs | 1 + pkg/debian/salt-common.install | 2 +- pkg/debian/salt-common.preinst | 4 ++++ pkg/rpm/salt.spec | 2 +- 6 files changed, 8 insertions(+), 2 deletions(-) rename pkg/common/{salt-common.logrotate => logrotate/salt-common} (100%) create mode 100644 pkg/debian/salt-common.conffiles diff --git a/pkg/common/salt-common.logrotate b/pkg/common/logrotate/salt-common similarity index 100% rename from pkg/common/salt-common.logrotate rename to pkg/common/logrotate/salt-common diff --git a/pkg/debian/salt-common.conffiles b/pkg/debian/salt-common.conffiles new file mode 100644 index 00000000000..595731d1d02 --- /dev/null +++ b/pkg/debian/salt-common.conffiles @@ -0,0 +1 @@ +/etc/logrotate.d/salt-common diff --git a/pkg/debian/salt-common.dirs b/pkg/debian/salt-common.dirs index 5c6dd29e224..381ec1f48ce 100644 --- a/pkg/debian/salt-common.dirs +++ b/pkg/debian/salt-common.dirs @@ -4,3 +4,4 @@ /usr/share/fish/vendor_completions.d /opt/saltstack/salt /etc/salt +/etc/logrotate.d diff --git a/pkg/debian/salt-common.install b/pkg/debian/salt-common.install index 09bb9e28099..4f8dac552ec 100644 --- a/pkg/debian/salt-common.install +++ b/pkg/debian/salt-common.install @@ -1,4 +1,4 @@ -pkg/common/salt-common.logrotate /etc/logrotate.d/salt +pkg/common/logrotate/salt-common /etc/logrotate.d pkg/common/fish-completions/salt-cp.fish /usr/share/fish/vendor_completions.d pkg/common/fish-completions/salt-call.fish /usr/share/fish/vendor_completions.d pkg/common/fish-completions/salt-syndic.fish /usr/share/fish/vendor_completions.d diff --git a/pkg/debian/salt-common.preinst b/pkg/debian/salt-common.preinst index 0e4ce9c59f7..0e45d2399f6 100644 --- a/pkg/debian/salt-common.preinst +++ b/pkg/debian/salt-common.preinst @@ -31,5 +31,9 @@ case "$1" in -s $SALT_SHELL \ -g $SALT_GROUP \ $SALT_USER + + # Remove incorrectly installed logrotate config - issue 65231 + test -d /etc/logrotate.d/salt && rm -r /etc/logrotate.d/salt || /bin/true + ;; esac diff --git a/pkg/rpm/salt.spec b/pkg/rpm/salt.spec index 6cc4e85c23c..4659c9fd343 100644 --- a/pkg/rpm/salt.spec +++ b/pkg/rpm/salt.spec @@ -266,7 +266,7 @@ install -p -m 0644 %{_salt_src}/pkg/common/salt-proxy@.service %{buildroot}%{_un # Logrotate #install -p %{SOURCE10} . mkdir -p %{buildroot}%{_sysconfdir}/logrotate.d/ -install -p -m 0644 %{_salt_src}/pkg/common/salt-common.logrotate %{buildroot}%{_sysconfdir}/logrotate.d/salt +install -p -m 0644 %{_salt_src}/pkg/common/logrotate/salt-common %{buildroot}%{_sysconfdir}/logrotate.d/salt # Bash completion mkdir -p %{buildroot}%{_sysconfdir}/bash_completion.d/ From 59f731c94fb6db7bb2a16f5971b1c200a579d3ac Mon Sep 17 00:00:00 2001 From: Barney Sowood Date: Thu, 12 Oct 2023 19:04:51 +0100 Subject: [PATCH 16/26] Add initial tests for logrotate configs Adds initial tests for logrotate configs to ensure they are installed in the correct location. --- .../integration/test_logrotate_config.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 pkg/tests/integration/test_logrotate_config.py diff --git a/pkg/tests/integration/test_logrotate_config.py b/pkg/tests/integration/test_logrotate_config.py new file mode 100644 index 00000000000..9179ae631d6 --- /dev/null +++ b/pkg/tests/integration/test_logrotate_config.py @@ -0,0 +1,34 @@ +""" +Tests for logrotate config +""" + +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.skip_on_windows, pytest.mark.skip_on_darwin] + + +@pytest.fixture +def logrotate_config_file(salt_call_cli): + """ + Fixture for logrotate config file path + """ + logrotate_dir = Path("/etc/logrotate.d") + + os_family = salt_call_cli.run("grains.get", "os_family") + assert os_family.returncode == 0 + + if os_family.data == "RedHat": + return logrotate_dir / "salt" + elif os_family.data == "Debian": + return logrotate_dir / "salt-common" + + +def test_logrotate_config(logrotate_config_file): + """ + Test that logrotate config has been installed in correctly + """ + assert logrotate_config_file.is_file() + assert logrotate_config_file.owner() == "root" + assert logrotate_config_file.group() == "root" From 1dad8745de53575bcfa3a83bc28c21e82fd18715 Mon Sep 17 00:00:00 2001 From: Barney Sowood Date: Thu, 12 Oct 2023 19:52:43 +0100 Subject: [PATCH 17/26] Add test for issue 65231 --- pkg/tests/integration/test_logrotate_config.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/tests/integration/test_logrotate_config.py b/pkg/tests/integration/test_logrotate_config.py index 9179ae631d6..5581105f11f 100644 --- a/pkg/tests/integration/test_logrotate_config.py +++ b/pkg/tests/integration/test_logrotate_config.py @@ -32,3 +32,12 @@ def test_logrotate_config(logrotate_config_file): assert logrotate_config_file.is_file() assert logrotate_config_file.owner() == "root" assert logrotate_config_file.group() == "root" + + +def test_issue_65231_etc_logrotate_salt_dir_removed(): + """ + Test that /etc/logrotate.d/salt is not a directory + """ + path = Path("/etc/logrotate.d/salt") + if path.exists(): + assert path.is_dir() is False From ffb8e84f60089cf381485fc8f238a1eb8840f263 Mon Sep 17 00:00:00 2001 From: Barney Sowood Date: Thu, 12 Oct 2023 20:21:19 +0100 Subject: [PATCH 18/26] Add changelog --- changelog/65231.fixed.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog/65231.fixed.md diff --git a/changelog/65231.fixed.md b/changelog/65231.fixed.md new file mode 100644 index 00000000000..50d225e7452 --- /dev/null +++ b/changelog/65231.fixed.md @@ -0,0 +1,2 @@ +Install logrotate config as /etc/logrotate.d/salt-common for Debian packages +Remove broken /etc/logrotate.d/salt directory from 3006.3 if it exists. From e5b94897310c2d1a240ad82fd1e59cd33bed8a6d Mon Sep 17 00:00:00 2001 From: Barney Sowood Date: Thu, 19 Oct 2023 21:50:41 +0100 Subject: [PATCH 19/26] Disable test for 65231 on downgrade to 3006.3 Disables the running of the test for 65231 when downgrading to 3006.3 as the issue that is fixed is in 3006.3 so test will fail when 3006.3 is installed. --- pkg/tests/integration/test_logrotate_config.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/tests/integration/test_logrotate_config.py b/pkg/tests/integration/test_logrotate_config.py index 5581105f11f..0ce9552546c 100644 --- a/pkg/tests/integration/test_logrotate_config.py +++ b/pkg/tests/integration/test_logrotate_config.py @@ -4,6 +4,7 @@ Tests for logrotate config from pathlib import Path +import packaging.version import pytest pytestmark = [pytest.mark.skip_on_windows, pytest.mark.skip_on_darwin] @@ -34,10 +35,15 @@ def test_logrotate_config(logrotate_config_file): assert logrotate_config_file.group() == "root" -def test_issue_65231_etc_logrotate_salt_dir_removed(): +def test_issue_65231_etc_logrotate_salt_dir_removed(install_salt): """ Test that /etc/logrotate.d/salt is not a directory """ + if install_salt.prev_version and packaging.version.parse( + install_salt.prev_version + ) == packaging.version.parse("3006.3"): + pytest.skip("Testing a downgrade to 3006.3, do not run") + path = Path("/etc/logrotate.d/salt") if path.exists(): assert path.is_dir() is False From b83ede52726793741fe4f80512e60fe7749a142f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 2 Nov 2023 12:54:02 +0000 Subject: [PATCH 20/26] Address my own review comments Signed-off-by: Pedro Algarvio --- .../integration/test_logrotate_config.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/pkg/tests/integration/test_logrotate_config.py b/pkg/tests/integration/test_logrotate_config.py index 0ce9552546c..2a1cb7cd334 100644 --- a/pkg/tests/integration/test_logrotate_config.py +++ b/pkg/tests/integration/test_logrotate_config.py @@ -2,28 +2,25 @@ Tests for logrotate config """ -from pathlib import Path +import pathlib import packaging.version import pytest -pytestmark = [pytest.mark.skip_on_windows, pytest.mark.skip_on_darwin] +pytestmark = [ + pytest.mark.skip_unless_on_linux, +] @pytest.fixture -def logrotate_config_file(salt_call_cli): +def logrotate_config_file(grains): """ Fixture for logrotate config file path """ - logrotate_dir = Path("/etc/logrotate.d") - - os_family = salt_call_cli.run("grains.get", "os_family") - assert os_family.returncode == 0 - - if os_family.data == "RedHat": - return logrotate_dir / "salt" - elif os_family.data == "Debian": - return logrotate_dir / "salt-common" + if grains["os_family"] == "RedHat": + return pathlib.Path("/etc/logrotate.d", "salt") + elif grains["os_family"] == "Debian": + return pathlib.Path("/etc/logrotate.d", "salt-common") def test_logrotate_config(logrotate_config_file): @@ -44,6 +41,6 @@ def test_issue_65231_etc_logrotate_salt_dir_removed(install_salt): ) == packaging.version.parse("3006.3"): pytest.skip("Testing a downgrade to 3006.3, do not run") - path = Path("/etc/logrotate.d/salt") + path = pathlib.Path("/etc/logrotate.d/salt") if path.exists(): assert path.is_dir() is False From 2ba6543016d5e3f93e14171c55a8ff8769600731 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 2 Nov 2023 16:22:14 +0000 Subject: [PATCH 21/26] The problem is still present in 3006.4, can't test downgrade Signed-off-by: Pedro Algarvio --- pkg/tests/integration/test_logrotate_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/tests/integration/test_logrotate_config.py b/pkg/tests/integration/test_logrotate_config.py index 2a1cb7cd334..fea0123b6eb 100644 --- a/pkg/tests/integration/test_logrotate_config.py +++ b/pkg/tests/integration/test_logrotate_config.py @@ -38,8 +38,8 @@ def test_issue_65231_etc_logrotate_salt_dir_removed(install_salt): """ if install_salt.prev_version and packaging.version.parse( install_salt.prev_version - ) == packaging.version.parse("3006.3"): - pytest.skip("Testing a downgrade to 3006.3, do not run") + ) <= packaging.version.parse("3006.4"): + pytest.skip("Testing a downgrade to 3006.4, do not run") path = pathlib.Path("/etc/logrotate.d/salt") if path.exists(): From 480a543db7a148cb2a89ca711537592ba0ca85e8 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 3 Nov 2023 05:47:57 +0000 Subject: [PATCH 22/26] 5 hours is not enough to run the integration tests in a single chunk Signed-off-by: Pedro Algarvio --- .github/workflows/nightly.yml | 58 +++++++++---------- .github/workflows/scheduled.yml | 58 +++++++++---------- .../workflows/templates/test-salt.yml.jinja | 2 +- .github/workflows/test-action-macos.yml | 8 +-- .github/workflows/test-action.yml | 8 +-- 5 files changed, 67 insertions(+), 67 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9224c915223..41108ef0e10 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1909,7 +1909,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 windows-2019: name: Windows 2019 Test @@ -1931,7 +1931,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 windows-2022: name: Windows 2022 Test @@ -1953,7 +1953,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 macos-12: name: macOS 12 Test @@ -1975,7 +1975,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 almalinux-8: name: Alma Linux 8 Test @@ -1997,7 +1997,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 almalinux-9: name: Alma Linux 9 Test @@ -2019,7 +2019,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 amazonlinux-2: name: Amazon Linux 2 Test @@ -2041,7 +2041,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 archlinux-lts: name: Arch Linux LTS Test @@ -2063,7 +2063,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 centos-7: name: CentOS 7 Test @@ -2085,7 +2085,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 centosstream-8: name: CentOS Stream 8 Test @@ -2107,7 +2107,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 centosstream-9: name: CentOS Stream 9 Test @@ -2129,7 +2129,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 debian-10: name: Debian 10 Test @@ -2151,7 +2151,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 debian-11: name: Debian 11 Test @@ -2173,7 +2173,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 debian-11-arm64: name: Debian 11 Arm64 Test @@ -2195,7 +2195,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 debian-12: name: Debian 12 Test @@ -2217,7 +2217,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 debian-12-arm64: name: Debian 12 Arm64 Test @@ -2239,7 +2239,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 fedora-37: name: Fedora 37 Test @@ -2261,7 +2261,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 fedora-38: name: Fedora 38 Test @@ -2283,7 +2283,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 opensuse-15: name: Opensuse 15 Test @@ -2305,7 +2305,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 photonos-3: name: Photon OS 3 Test @@ -2327,7 +2327,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 photonos-3-arm64: name: Photon OS 3 Arm64 Test @@ -2349,7 +2349,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 photonos-4: name: Photon OS 4 Test @@ -2371,7 +2371,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 photonos-4-arm64: name: Photon OS 4 Arm64 Test @@ -2393,7 +2393,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 photonos-5: name: Photon OS 5 Test @@ -2415,7 +2415,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 photonos-5-arm64: name: Photon OS 5 Arm64 Test @@ -2437,7 +2437,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 ubuntu-2004: name: Ubuntu 20.04 Test @@ -2459,7 +2459,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 ubuntu-2004-arm64: name: Ubuntu 20.04 Arm64 Test @@ -2481,7 +2481,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 ubuntu-2204: name: Ubuntu 22.04 Test @@ -2503,7 +2503,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 ubuntu-2204-arm64: name: Ubuntu 22.04 Arm64 Test @@ -2525,7 +2525,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: nightly - default-timeout: 300 + default-timeout: 360 combine-all-code-coverage: name: Combine Code Coverage diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 287dd7fa7a8..8feb2ec7870 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -1882,7 +1882,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 windows-2019: name: Windows 2019 Test @@ -1904,7 +1904,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 windows-2022: name: Windows 2022 Test @@ -1926,7 +1926,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 macos-12: name: macOS 12 Test @@ -1948,7 +1948,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 almalinux-8: name: Alma Linux 8 Test @@ -1970,7 +1970,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 almalinux-9: name: Alma Linux 9 Test @@ -1992,7 +1992,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 amazonlinux-2: name: Amazon Linux 2 Test @@ -2014,7 +2014,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 archlinux-lts: name: Arch Linux LTS Test @@ -2036,7 +2036,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 centos-7: name: CentOS 7 Test @@ -2058,7 +2058,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 centosstream-8: name: CentOS Stream 8 Test @@ -2080,7 +2080,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 centosstream-9: name: CentOS Stream 9 Test @@ -2102,7 +2102,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 debian-10: name: Debian 10 Test @@ -2124,7 +2124,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 debian-11: name: Debian 11 Test @@ -2146,7 +2146,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 debian-11-arm64: name: Debian 11 Arm64 Test @@ -2168,7 +2168,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 debian-12: name: Debian 12 Test @@ -2190,7 +2190,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 debian-12-arm64: name: Debian 12 Arm64 Test @@ -2212,7 +2212,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 fedora-37: name: Fedora 37 Test @@ -2234,7 +2234,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 fedora-38: name: Fedora 38 Test @@ -2256,7 +2256,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 opensuse-15: name: Opensuse 15 Test @@ -2278,7 +2278,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 photonos-3: name: Photon OS 3 Test @@ -2300,7 +2300,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 photonos-3-arm64: name: Photon OS 3 Arm64 Test @@ -2322,7 +2322,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 photonos-4: name: Photon OS 4 Test @@ -2344,7 +2344,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 photonos-4-arm64: name: Photon OS 4 Arm64 Test @@ -2366,7 +2366,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 photonos-5: name: Photon OS 5 Test @@ -2388,7 +2388,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 photonos-5-arm64: name: Photon OS 5 Arm64 Test @@ -2410,7 +2410,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 ubuntu-2004: name: Ubuntu 20.04 Test @@ -2432,7 +2432,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 ubuntu-2004-arm64: name: Ubuntu 20.04 Arm64 Test @@ -2454,7 +2454,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 ubuntu-2204: name: Ubuntu 22.04 Test @@ -2476,7 +2476,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 ubuntu-2204-arm64: name: Ubuntu 22.04 Arm64 Test @@ -2498,7 +2498,7 @@ jobs: skip-code-coverage: false skip-junit-reports: false workflow-slug: scheduled - default-timeout: 300 + default-timeout: 360 combine-all-code-coverage: name: Combine Code Coverage diff --git a/.github/workflows/templates/test-salt.yml.jinja b/.github/workflows/templates/test-salt.yml.jinja index ce00a6efd1f..cade2fe631c 100644 --- a/.github/workflows/templates/test-salt.yml.jinja +++ b/.github/workflows/templates/test-salt.yml.jinja @@ -1,5 +1,5 @@ <%- if workflow_slug in ("nightly", "scheduled") %> - <%- set timeout_value = 300 %> + <%- set timeout_value = 360 %> <%- else %> <%- set timeout_value = 120 %> <%- endif %> diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index ec84dec71c2..6eb610302c0 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -64,8 +64,8 @@ on: default-timeout: required: false type: number - description: Timeout, in minutes, for the test job(Default 300, 5 hours). - default: 300 + description: Timeout, in minutes, for the test job(Default 360, 6 hours). + default: 360 env: COLUMNS: 190 @@ -101,8 +101,8 @@ jobs: name: Test runs-on: ${{ inputs.distro-slug }} # Full test runs. Each chunk should never take more than 2 hours. - # Partial test runs(no chunk parallelization), 5 Hours - timeout-minutes: ${{ fromJSON(inputs.testrun)['type'] == 'full' && inputs.default-timeout || 300 }} + # Partial test runs(no chunk parallelization), 6 Hours + timeout-minutes: ${{ fromJSON(inputs.testrun)['type'] == 'full' && inputs.default-timeout || 360 }} needs: - generate-matrix strategy: diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index 0f9fdf2b1e2..53e7bbfa894 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -64,8 +64,8 @@ on: default-timeout: required: false type: number - description: Timeout, in minutes, for the test job(Default 300, 5 hours). - default: 300 + description: Timeout, in minutes, for the test job(Default 360, 6 hours). + default: 360 env: COLUMNS: 190 @@ -109,8 +109,8 @@ jobs: - linux - bastion # Full test runs. Each chunk should never take more than 2 hours. - # Partial test runs(no chunk parallelization), 5 Hours - timeout-minutes: ${{ fromJSON(inputs.testrun)['type'] == 'full' && inputs.default-timeout || 300 }} + # Partial test runs(no chunk parallelization), 6 Hours + timeout-minutes: ${{ fromJSON(inputs.testrun)['type'] == 'full' && inputs.default-timeout || 360 }} needs: - generate-matrix strategy: From bb1bc2a326c1f1dfcb3b4f7c1a8aacea22e18db4 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 3 Nov 2023 05:51:55 +0000 Subject: [PATCH 23/26] And apparently 2 hours is not enough to run one of the integration test chunks Signed-off-by: Pedro Algarvio --- .github/workflows/ci.yml | 58 +++++++++---------- .github/workflows/staging.yml | 58 +++++++++---------- .../workflows/templates/test-salt.yml.jinja | 2 +- 3 files changed, 59 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a105e974f8a..1c2ff7aa2da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1848,7 +1848,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 windows-2019: name: Windows 2019 Test @@ -1870,7 +1870,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 windows-2022: name: Windows 2022 Test @@ -1892,7 +1892,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 macos-12: name: macOS 12 Test @@ -1914,7 +1914,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 almalinux-8: name: Alma Linux 8 Test @@ -1936,7 +1936,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 almalinux-9: name: Alma Linux 9 Test @@ -1958,7 +1958,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 amazonlinux-2: name: Amazon Linux 2 Test @@ -1980,7 +1980,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 archlinux-lts: name: Arch Linux LTS Test @@ -2002,7 +2002,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 centos-7: name: CentOS 7 Test @@ -2024,7 +2024,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 centosstream-8: name: CentOS Stream 8 Test @@ -2046,7 +2046,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 centosstream-9: name: CentOS Stream 9 Test @@ -2068,7 +2068,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 debian-10: name: Debian 10 Test @@ -2090,7 +2090,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 debian-11: name: Debian 11 Test @@ -2112,7 +2112,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 debian-11-arm64: name: Debian 11 Arm64 Test @@ -2134,7 +2134,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 debian-12: name: Debian 12 Test @@ -2156,7 +2156,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 debian-12-arm64: name: Debian 12 Arm64 Test @@ -2178,7 +2178,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 fedora-37: name: Fedora 37 Test @@ -2200,7 +2200,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 fedora-38: name: Fedora 38 Test @@ -2222,7 +2222,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 opensuse-15: name: Opensuse 15 Test @@ -2244,7 +2244,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 photonos-3: name: Photon OS 3 Test @@ -2266,7 +2266,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 photonos-3-arm64: name: Photon OS 3 Arm64 Test @@ -2288,7 +2288,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 photonos-4: name: Photon OS 4 Test @@ -2310,7 +2310,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 photonos-4-arm64: name: Photon OS 4 Arm64 Test @@ -2332,7 +2332,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 photonos-5: name: Photon OS 5 Test @@ -2354,7 +2354,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 photonos-5-arm64: name: Photon OS 5 Arm64 Test @@ -2376,7 +2376,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 ubuntu-2004: name: Ubuntu 20.04 Test @@ -2398,7 +2398,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 ubuntu-2004-arm64: name: Ubuntu 20.04 Arm64 Test @@ -2420,7 +2420,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 ubuntu-2204: name: Ubuntu 22.04 Test @@ -2442,7 +2442,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 ubuntu-2204-arm64: name: Ubuntu 22.04 Arm64 Test @@ -2464,7 +2464,7 @@ jobs: skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} skip-junit-reports: ${{ github.event_name == 'pull_request' }} workflow-slug: ci - default-timeout: 120 + default-timeout: 180 combine-all-code-coverage: name: Combine Code Coverage diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index afdae3bd2f1..fdad325bee2 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -1904,7 +1904,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 windows-2019: name: Windows 2019 Test @@ -1926,7 +1926,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 windows-2022: name: Windows 2022 Test @@ -1948,7 +1948,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 macos-12: name: macOS 12 Test @@ -1970,7 +1970,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 almalinux-8: name: Alma Linux 8 Test @@ -1992,7 +1992,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 almalinux-9: name: Alma Linux 9 Test @@ -2014,7 +2014,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 amazonlinux-2: name: Amazon Linux 2 Test @@ -2036,7 +2036,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 archlinux-lts: name: Arch Linux LTS Test @@ -2058,7 +2058,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 centos-7: name: CentOS 7 Test @@ -2080,7 +2080,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 centosstream-8: name: CentOS Stream 8 Test @@ -2102,7 +2102,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 centosstream-9: name: CentOS Stream 9 Test @@ -2124,7 +2124,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 debian-10: name: Debian 10 Test @@ -2146,7 +2146,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 debian-11: name: Debian 11 Test @@ -2168,7 +2168,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 debian-11-arm64: name: Debian 11 Arm64 Test @@ -2190,7 +2190,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 debian-12: name: Debian 12 Test @@ -2212,7 +2212,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 debian-12-arm64: name: Debian 12 Arm64 Test @@ -2234,7 +2234,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 fedora-37: name: Fedora 37 Test @@ -2256,7 +2256,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 fedora-38: name: Fedora 38 Test @@ -2278,7 +2278,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 opensuse-15: name: Opensuse 15 Test @@ -2300,7 +2300,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 photonos-3: name: Photon OS 3 Test @@ -2322,7 +2322,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 photonos-3-arm64: name: Photon OS 3 Arm64 Test @@ -2344,7 +2344,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 photonos-4: name: Photon OS 4 Test @@ -2366,7 +2366,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 photonos-4-arm64: name: Photon OS 4 Arm64 Test @@ -2388,7 +2388,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 photonos-5: name: Photon OS 5 Test @@ -2410,7 +2410,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 photonos-5-arm64: name: Photon OS 5 Arm64 Test @@ -2432,7 +2432,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 ubuntu-2004: name: Ubuntu 20.04 Test @@ -2454,7 +2454,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 ubuntu-2004-arm64: name: Ubuntu 20.04 Arm64 Test @@ -2476,7 +2476,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 ubuntu-2204: name: Ubuntu 22.04 Test @@ -2498,7 +2498,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 ubuntu-2204-arm64: name: Ubuntu 22.04 Arm64 Test @@ -2520,7 +2520,7 @@ jobs: skip-code-coverage: true skip-junit-reports: true workflow-slug: staging - default-timeout: 120 + default-timeout: 180 build-src-repo: name: Build Repository diff --git a/.github/workflows/templates/test-salt.yml.jinja b/.github/workflows/templates/test-salt.yml.jinja index cade2fe631c..be3e7fd5fa4 100644 --- a/.github/workflows/templates/test-salt.yml.jinja +++ b/.github/workflows/templates/test-salt.yml.jinja @@ -1,7 +1,7 @@ <%- if workflow_slug in ("nightly", "scheduled") %> <%- set timeout_value = 360 %> <%- else %> - <%- set timeout_value = 120 %> + <%- set timeout_value = 180 %> <%- endif %> <%- for slug, display_name, arch in test_salt_listing["windows"] %> From 45932d911288bb3212a6ffe2e093d7e1bd1f92b6 Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Wed, 1 Nov 2023 16:58:26 -0600 Subject: [PATCH 24/26] WIP Ported test_data.py from unittest to pytest and adding additional tests --- tests/pytests/unit/utils/test_data.py | 1536 ++++++++++++++++++++++++- tests/unit/utils/test_data.py | 1506 ------------------------ 2 files changed, 1473 insertions(+), 1569 deletions(-) delete mode 100644 tests/unit/utils/test_data.py diff --git a/tests/pytests/unit/utils/test_data.py b/tests/pytests/unit/utils/test_data.py index 61d834da7f1..01f9ae3cfe5 100644 --- a/tests/pytests/unit/utils/test_data.py +++ b/tests/pytests/unit/utils/test_data.py @@ -1,89 +1,1499 @@ +""" +Tests for salt.utils.data +""" +import builtins +import logging + import pytest import salt.utils.data +import salt.utils.stringutils +from salt.utils.odict import OrderedDict as SaltOrderedDict +from tests.support.mock import patch +from tests.support.unit import LOREM_IPSUM + +log = logging.getLogger(__name__) + +_b = lambda x: x.encode("utf-8") +_s = lambda x: salt.utils.stringutils.to_str(x, normalize=True) -def test_get_value_simple_path(): - data = {"a": {"b": {"c": "foo"}}} - assert [{"value": "foo"}] == salt.utils.data.get_value(data, "a:b:c") +@pytest.fixture +def get_BYTES(): + # Some randomized data that will not decode + return b"1\x814\x10" -def test_get_value_placeholder_dict(): - data = {"a": {"b": {"name": "foo"}, "c": {"name": "bar"}}} - assert [ - {"value": "foo", "id": "b"}, - {"value": "bar", "id": "c"}, - ] == salt.utils.data.get_value(data, "a:{id}:name") +@pytest.fixture +def get_EGGS(): + # This is an example of a unicode string with й constructed using two separate + # code points. Do not modify it. + return "\u044f\u0438\u0306\u0446\u0430" -def test_get_value_placeholder_list(): - data = {"a": [{"name": "foo"}, {"name": "bar"}]} - assert [ - {"value": "foo", "id": 0}, - {"value": "bar", "id": 1}, - ] == salt.utils.data.get_value(data, "a:{id}:name") +@pytest.fixture +def get_test_data(get_BYTES, get_EGGS): + return [ + "unicode_str", + _b("питон"), + 123, + 456.789, + True, + False, + None, + get_EGGS, + get_BYTES, + [123, 456.789, _b("спам"), True, False, None, get_EGGS, get_BYTES], + (987, 654.321, _b("яйца"), get_EGGS, None, (True, get_EGGS, get_BYTES)), + { + _b("str_key"): _b("str_val"), + None: True, + 123: 456.789, + get_EGGS: get_BYTES, + _b("subdict"): { + "unicode_key": get_EGGS, + _b("tuple"): (123, "hello", _b("world"), True, get_EGGS, get_BYTES), + _b("list"): [456, _b("спам"), False, get_EGGS, get_BYTES], + }, + }, + SaltOrderedDict([(_b("foo"), "bar"), (123, 456), (get_EGGS, get_BYTES)]), + ] -def test_get_value_nested_placeholder(): - data = { - "a": { - "b": {"b1": {"name": "foo1"}, "b2": {"name": "foo2"}}, - "c": {"c1": {"name": "bar"}}, +def test_sorted_ignorecase(): + test_list = ["foo", "Foo", "bar", "Bar"] + expected_list = ["bar", "Bar", "foo", "Foo"] + assert salt.utils.data.sorted_ignorecase(test_list) == expected_list + + +def test_mysql_to_dict(): + test_mysql_output = [ + "+----+------+-----------+------+---------+------+-------+------------------+", + "| Id | User | Host | db | Command | Time | State | Info " + " |", + "+----+------+-----------+------+---------+------+-------+------------------+", + "| 7 | root | localhost | NULL | Query | 0 | init | show" + " processlist |", + "+----+------+-----------+------+---------+------+-------+------------------+", + ] + + ret = salt.utils.data.mysql_to_dict(test_mysql_output, "Info") + expected_dict = { + "show processlist": { + "Info": "show processlist", + "db": "NULL", + "State": "init", + "Host": "localhost", + "Command": "Query", + "User": "root", + "Time": 0, + "Id": 7, } } - assert [ - {"value": "foo1", "id": "b", "sub": "b1"}, - {"value": "foo2", "id": "b", "sub": "b2"}, - {"value": "bar", "id": "c", "sub": "c1"}, - ] == salt.utils.data.get_value(data, "a:{id}:{sub}:name") + assert ret == expected_dict -def test_get_value_nested_notfound(): - data = {"a": {"b": {"c": "foo"}}} - assert [{"value": []}] == salt.utils.data.get_value(data, "a:b:d", []) +def test_subdict_match(): + test_two_level_dict = {"foo": {"bar": "baz"}} + test_two_level_comb_dict = {"foo": {"bar": "baz:woz"}} + test_two_level_dict_and_list = { + "abc": ["def", "ghi", {"lorem": {"ipsum": [{"dolor": "sit"}]}}], + } + test_three_level_dict = {"a": {"b": {"c": "v"}}} + + assert salt.utils.data.subdict_match(test_two_level_dict, "foo:bar:baz") + # In test_two_level_comb_dict, 'foo:bar' corresponds to 'baz:woz', not + # 'baz'. This match should return False. + assert not salt.utils.data.subdict_match(test_two_level_comb_dict, "foo:bar:baz") + # This tests matching with the delimiter in the value part (in other + # words, that the path 'foo:bar' corresponds to the string 'baz:woz'). + assert salt.utils.data.subdict_match(test_two_level_comb_dict, "foo:bar:baz:woz") + # This would match if test_two_level_comb_dict['foo']['bar'] was equal + # to 'baz:woz:wiz', or if there was more deep nesting. But it does not, + # so this should return False. + assert not salt.utils.data.subdict_match( + test_two_level_comb_dict, "foo:bar:baz:woz:wiz" + ) + # This tests for cases when a key path corresponds to a list. The + # value part 'ghi' should be successfully matched as it is a member of + # the list corresponding to key path 'abc'. It is somewhat a + # duplication of a test within test_traverse_dict_and_list, but + # salt.utils.data.subdict_match() does more than just invoke + # salt.utils.traverse_list_and_dict() so this particular assertion is a + # sanity check. + assert salt.utils.data.subdict_match(test_two_level_dict_and_list, "abc:ghi") + # This tests the use case of a dict embedded in a list, embedded in a + # list, embedded in a dict. This is a rather absurd case, but it + # confirms that match recursion works properly. + assert salt.utils.data.subdict_match( + test_two_level_dict_and_list, "abc:lorem:ipsum:dolor:sit" + ) + # Test four level dict match for reference + assert salt.utils.data.subdict_match(test_three_level_dict, "a:b:c:v") + # Test regression in 2015.8 where 'a:c:v' would match 'a:b:c:v' + assert not salt.utils.data.subdict_match(test_three_level_dict, "a:c:v") + # Test wildcard match + assert salt.utils.data.subdict_match(test_three_level_dict, "a:*:c:v") -def test_get_value_not_found(): - assert [{"value": []}] == salt.utils.data.get_value({}, "a", []) - - -def test_get_value_none(): - assert [{"value": None}] == salt.utils.data.get_value({"a": None}, "a") - - -def test_get_value_simple_type_path(): - assert [{"value": []}] == salt.utils.data.get_value({"a": 1024}, "a:b", []) - - -def test_get_value_None_path(): - assert [{"value": None}] == salt.utils.data.get_value({"a": None}, "a:b", []) - - -def test_flatten_recursion_error(): +@pytest.mark.parametrize( + "wildcard", + [ + ("*:*:*:*"), + ("a:*:*:*"), + ("a:b:*:*"), + ("a:b:ç:*"), + ("a:b:*:d"), + ("a:*:ç:d"), + ("*:b:ç:d"), + ("*:*:ç:d"), + ("*:*:*:d"), + ("a:*:*:d"), + ("a:b:*:ef*"), + ("a:b:*:g*"), + ("a:b:*:j:*"), + ("a:b:*:j:k"), + ("a:b:*:*:k"), + ("a:b:*:*:*"), + ], +) +def test_subdict_match_with_wildcards(wildcard): """ - Test the flatten function for reference cycle detection + Tests subdict matching when wildcards are used in the expression """ - data = [1, 2, 3, [4]] - data.append(data) - with pytest.raises(RecursionError) as err: - salt.utils.data.flatten(data) - assert str(err.value) == "Reference cycle detected. Check input list." + data = {"a": {"b": {"ç": "d", "é": ["eff", "gee", "8ch"], "ĩ": {"j": "k"}}}} + assert salt.utils.data.subdict_match(data, wildcard) -def test_sample(): - lst = ["one", "two", "three", "four"] - assert len(salt.utils.data.sample(lst, 0)) == 0 - assert len(salt.utils.data.sample(lst, 2)) == 2 - pytest.raises(ValueError, salt.utils.data.sample, lst, 5) - assert salt.utils.data.sample(lst, 2, seed="static") == ["four", "two"] +def test_traverse_dict(): + test_two_level_dict = {"foo": {"bar": "baz"}} + + assert {"not_found": "nope"} == salt.utils.data.traverse_dict( + test_two_level_dict, "foo:bar:baz", {"not_found": "nope"} + ) + assert "baz" == salt.utils.data.traverse_dict( + test_two_level_dict, "foo:bar", {"not_found": "not_found"} + ) -def test_shuffle(): - lst = ["one", "two", "three", "four"] - assert len(salt.utils.data.shuffle(lst)) == 4 - assert salt.utils.data.shuffle(lst, seed="static") == [ - "four", +def test_traverse_dict_and_list(): + test_two_level_dict = {"foo": {"bar": "baz"}} + test_two_level_dict_and_list = { + "foo": ["bar", "baz", {"lorem": {"ipsum": [{"dolor": "sit"}]}}] + } + + # Check traversing too far: salt.utils.data.traverse_dict_and_list() returns + # the value corresponding to a given key path, and baz is a value + # corresponding to the key path foo:bar. + assert {"not_found": "nope"} == salt.utils.data.traverse_dict_and_list( + test_two_level_dict, "foo:bar:baz", {"not_found": "nope"} + ) + # Now check to ensure that foo:bar corresponds to baz + assert "baz" == salt.utils.data.traverse_dict_and_list( + test_two_level_dict, "foo:bar", {"not_found": "not_found"} + ) + # Check traversing too far + assert {"not_found": "nope"} == salt.utils.data.traverse_dict_and_list( + test_two_level_dict_and_list, "foo:bar", {"not_found": "nope"} + ) + # Check index 1 (2nd element) of list corresponding to path 'foo' + assert "baz" == salt.utils.data.traverse_dict_and_list( + test_two_level_dict_and_list, "foo:1", {"not_found": "not_found"} + ) + # Traverse a couple times into dicts embedded in lists + assert "sit" == salt.utils.data.traverse_dict_and_list( + test_two_level_dict_and_list, + "foo:lorem:ipsum:dolor", + {"not_found": "not_found"}, + ) + + # Traverse and match integer key in a nested dict + # https://github.com/saltstack/salt/issues/56444 + assert "it worked" == salt.utils.data.traverse_dict_and_list( + {"foo": {1234: "it worked"}}, + "foo:1234", + "it didn't work", + ) + # Make sure that we properly return the default value when the initial + # attempt fails and YAML-loading the target key doesn't change its + # value. + assert "default" == salt.utils.data.traverse_dict_and_list( + {"foo": {"baz": "didn't work"}}, + "foo:bar", + "default", + ) + + +def test_issue_39709(): + test_two_level_dict_and_list = { + "foo": ["bar", "baz", {"lorem": {"ipsum": [{"dolor": "sit"}]}}] + } + + assert "sit" == salt.utils.data.traverse_dict_and_list( + test_two_level_dict_and_list, + ["foo", "lorem", "ipsum", "dolor"], + {"not_found": "not_found"}, + ) + + +def test_compare_dicts(): + ret = salt.utils.data.compare_dicts(old={"foo": "bar"}, new={"foo": "bar"}) + assert ret == {} + + ret = salt.utils.data.compare_dicts(old={"foo": "bar"}, new={"foo": "woz"}) + expected_ret = {"foo": {"new": "woz", "old": "bar"}} + assert ret == expected_ret + + +def test_compare_lists_no_change(): + ret = salt.utils.data.compare_lists( + old=[1, 2, 3, "a", "b", "c"], new=[1, 2, 3, "a", "b", "c"] + ) + expected = {} + assert ret == expected + + +def test_compare_lists_changes(): + ret = salt.utils.data.compare_lists( + old=[1, 2, 3, "a", "b", "c"], new=[1, 2, 4, "x", "y", "z"] + ) + expected = {"new": [4, "x", "y", "z"], "old": [3, "a", "b", "c"]} + assert ret == expected + + +def test_compare_lists_changes_new(): + ret = salt.utils.data.compare_lists(old=[1, 2, 3], new=[1, 2, 3, "x", "y", "z"]) + expected = {"new": ["x", "y", "z"]} + assert ret == expected + + +def test_compare_lists_changes_old(): + ret = salt.utils.data.compare_lists(old=[1, 2, 3, "a", "b", "c"], new=[1, 2, 3]) + expected = {"old": ["a", "b", "c"]} + assert ret == expected + + +def test_decode(get_test_data, get_BYTES, get_EGGS): + """ + Companion to test_decode_to_str, they should both be kept up-to-date + with one another. + + NOTE: This uses the lambda "_b" defined above in the global scope, + which encodes a string to a bytestring, assuming utf-8. + """ + expected = [ + "unicode_str", + "питон", + 123, + 456.789, + True, + False, + None, + "яйца", + get_BYTES, + [123, 456.789, "спам", True, False, None, "яйца", get_BYTES], + (987, 654.321, "яйца", "яйца", None, (True, "яйца", get_BYTES)), + { + "str_key": "str_val", + None: True, + 123: 456.789, + "яйца": get_BYTES, + "subdict": { + "unicode_key": "яйца", + "tuple": (123, "hello", "world", True, "яйца", get_BYTES), + "list": [456, "спам", False, "яйца", get_BYTES], + }, + }, + SaltOrderedDict([("foo", "bar"), (123, 456), ("яйца", get_BYTES)]), + ] + + ret = salt.utils.data.decode( + get_test_data, + keep=True, + normalize=True, + preserve_dict_class=True, + preserve_tuples=True, + ) + assert ret == expected + + # The binary data in the data structure should fail to decode, even + # using the fallback, and raise an exception. + pytest.raises( + UnicodeDecodeError, + salt.utils.data.decode, + get_test_data, + keep=False, + normalize=True, + preserve_dict_class=True, + preserve_tuples=True, + ) + + # Now munge the expected data so that we get what we would expect if we + # disable preservation of dict class and tuples + expected[10] = [987, 654.321, "яйца", "яйца", None, [True, "яйца", get_BYTES]] + expected[11]["subdict"]["tuple"] = [123, "hello", "world", True, "яйца", get_BYTES] + expected[12] = {"foo": "bar", 123: 456, "яйца": get_BYTES} + + ret = salt.utils.data.decode( + get_test_data, + keep=True, + normalize=True, + preserve_dict_class=False, + preserve_tuples=False, + ) + assert ret == expected + + # Now test single non-string, non-data-structure items, these should + # return the same value when passed to this function + for item in (123, 4.56, True, False, None): + log.debug("Testing decode of %s", item) + assert salt.utils.data.decode(item) == item + + # Test single strings (not in a data structure) + assert salt.utils.data.decode("foo") == "foo" + assert salt.utils.data.decode(_b("bar")) == "bar" + assert salt.utils.data.decode(get_EGGS, normalize=True) == "яйца" + assert salt.utils.data.decode(get_EGGS, normalize=False) == get_EGGS + + # Test binary blob + assert salt.utils.data.decode(get_BYTES, keep=True) == get_BYTES + pytest.raises(UnicodeDecodeError, salt.utils.data.decode, get_BYTES, keep=False) + + +def test_circular_refs_dicts(): + test_dict = {"key": "value", "type": "test1"} + test_dict["self"] = test_dict + ret = salt.utils.data._remove_circular_refs(ob=test_dict) + assert ret == {"key": "value", "type": "test1", "self": None} + + +def test_circular_refs_lists(): + test_list = { + "foo": [], + } + test_list["foo"].append((test_list,)) + ret = salt.utils.data._remove_circular_refs(ob=test_list) + assert ret == {"foo": [(None,)]} + + +def test_circular_refs_tuple(): + test_dup = {"foo": "string 1", "bar": "string 1", "ham": 1, "spam": 1} + ret = salt.utils.data._remove_circular_refs(ob=test_dup) + assert ret == {"foo": "string 1", "bar": "string 1", "ham": 1, "spam": 1} + + +def test_decode_to_str(get_test_data, get_BYTES): + """ + Companion to test_decode, they should both be kept up-to-date with one + another. + + NOTE: This uses the lambda "_s" defined above in the global scope, + which converts the string/bytestring to a str type. + """ + expected = [ + _s("unicode_str"), + _s("питон"), + 123, + 456.789, + True, + False, + None, + _s("яйца"), + get_BYTES, + [123, 456.789, _s("спам"), True, False, None, _s("яйца"), get_BYTES], + (987, 654.321, _s("яйца"), _s("яйца"), None, (True, _s("яйца"), get_BYTES)), + { + _s("str_key"): _s("str_val"), + None: True, + 123: 456.789, + _s("яйца"): get_BYTES, + _s("subdict"): { + _s("unicode_key"): _s("яйца"), + _s("tuple"): ( + 123, + _s("hello"), + _s("world"), + True, + _s("яйца"), + get_BYTES, + ), + _s("list"): [456, _s("спам"), False, _s("яйца"), get_BYTES], + }, + }, + SaltOrderedDict([(_s("foo"), _s("bar")), (123, 456), (_s("яйца"), get_BYTES)]), + ] + + ret = salt.utils.data.decode( + get_test_data, + keep=True, + normalize=True, + preserve_dict_class=True, + preserve_tuples=True, + to_str=True, + ) + assert ret == expected + + # The binary data in the data structure should fail to decode, even + # using the fallback, and raise an exception. + pytest.raises( + UnicodeDecodeError, + salt.utils.data.decode, + get_test_data, + keep=False, + normalize=True, + preserve_dict_class=True, + preserve_tuples=True, + to_str=True, + ) + + # Now munge the expected data so that we get what we would expect if we + # disable preservation of dict class and tuples + expected[10] = [ + 987, + 654.321, + _s("яйца"), + _s("яйца"), + None, + [True, _s("яйца"), get_BYTES], + ] + expected[11][_s("subdict")][_s("tuple")] = [ + 123, + _s("hello"), + _s("world"), + True, + _s("яйца"), + get_BYTES, + ] + expected[12] = {_s("foo"): _s("bar"), 123: 456, _s("яйца"): get_BYTES} + + ret = salt.utils.data.decode( + get_test_data, + keep=True, + normalize=True, + preserve_dict_class=False, + preserve_tuples=False, + to_str=True, + ) + assert ret == expected + + # Now test single non-string, non-data-structure items, these should + # return the same value when passed to this function + for item in (123, 4.56, True, False, None): + log.debug("Testing decode of %s", item) + assert salt.utils.data.decode(item, to_str=True) == item + + # Test single strings (not in a data structure) + assert salt.utils.data.decode("foo", to_str=True) == _s("foo") + assert salt.utils.data.decode(_b("bar"), to_str=True) == _s("bar") + + # Test binary blob + assert salt.utils.data.decode(get_BYTES, keep=True, to_str=True) == get_BYTES + pytest.raises( + UnicodeDecodeError, + salt.utils.data.decode, + get_BYTES, + keep=False, + to_str=True, + ) + + +def test_decode_fallback(): + """ + Test fallback to utf-8 + """ + with patch.object(builtins, "__salt_system_encoding__", "ascii"): + assert salt.utils.data.decode(_b("яйца")) == "яйца" + + +def test_encode(get_test_data, get_BYTES, get_EGGS): + """ + NOTE: This uses the lambda "_b" defined above in the global scope, + which encodes a string to a bytestring, assuming utf-8. + """ + expected = [ + _b("unicode_str"), + _b("питон"), + 123, + 456.789, + True, + False, + None, + _b(get_EGGS), + get_BYTES, + [123, 456.789, _b("спам"), True, False, None, _b(get_EGGS), get_BYTES], + (987, 654.321, _b("яйца"), _b(get_EGGS), None, (True, _b(get_EGGS), get_BYTES)), + { + _b("str_key"): _b("str_val"), + None: True, + 123: 456.789, + _b(get_EGGS): get_BYTES, + _b("subdict"): { + _b("unicode_key"): _b(get_EGGS), + _b("tuple"): ( + 123, + _b("hello"), + _b("world"), + True, + _b(get_EGGS), + get_BYTES, + ), + _b("list"): [456, _b("спам"), False, _b(get_EGGS), get_BYTES], + }, + }, + SaltOrderedDict( + [(_b("foo"), _b("bar")), (123, 456), (_b(get_EGGS), get_BYTES)] + ), + ] + + # Both keep=True and keep=False should work because the get_BYTES data is + # already bytes. + ret = salt.utils.data.encode( + get_test_data, keep=True, preserve_dict_class=True, preserve_tuples=True + ) + assert ret == expected + ret = salt.utils.data.encode( + get_test_data, keep=False, preserve_dict_class=True, preserve_tuples=True + ) + assert ret == expected + + # Now munge the expected data so that we get what we would expect if we + # disable preservation of dict class and tuples + expected[10] = [ + 987, + 654.321, + _b("яйца"), + _b(get_EGGS), + None, + [True, _b(get_EGGS), get_BYTES], + ] + expected[11][_b("subdict")][_b("tuple")] = [ + 123, + _b("hello"), + _b("world"), + True, + _b(get_EGGS), + get_BYTES, + ] + expected[12] = {_b("foo"): _b("bar"), 123: 456, _b(get_EGGS): get_BYTES} + + ret = salt.utils.data.encode( + get_test_data, keep=True, preserve_dict_class=False, preserve_tuples=False + ) + assert ret == expected + ret = salt.utils.data.encode( + get_test_data, keep=False, preserve_dict_class=False, preserve_tuples=False + ) + assert ret == expected + + # Now test single non-string, non-data-structure items, these should + # return the same value when passed to this function + for item in (123, 4.56, True, False, None): + log.debug("Testing encode of %s", item) + assert salt.utils.data.encode(item) == item + + # Test single strings (not in a data structure) + assert salt.utils.data.encode("foo") == _b("foo") + assert salt.utils.data.encode(_b("bar")) == _b("bar") + + # Test binary blob, nothing should happen even when keep=False since + # the data is already bytes + assert salt.utils.data.encode(get_BYTES, keep=True) == get_BYTES + assert salt.utils.data.encode(get_BYTES, keep=False) == get_BYTES + + +def test_encode_keep(): + """ + Whereas we tested the keep argument in test_decode, it is much easier + to do a more comprehensive test of keep in its own function where we + can force the encoding. + """ + unicode_str = "питон" + encoding = "ascii" + + # Test single string + assert salt.utils.data.encode(unicode_str, encoding, keep=True) == unicode_str + pytest.raises( + UnicodeEncodeError, + salt.utils.data.encode, + unicode_str, + encoding, + keep=False, + ) + + data = [ + unicode_str, + [b"foo", [unicode_str], {b"key": unicode_str}, (unicode_str,)], + { + b"list": [b"foo", unicode_str], + b"dict": {b"key": unicode_str}, + b"tuple": (b"foo", unicode_str), + }, + ([b"foo", unicode_str], {b"key": unicode_str}, (unicode_str,)), + ] + + # Since everything was a bytestring aside from the bogus data, the + # return data should be identical. We don't need to test recursive + # decoding, that has already been tested in test_encode. + assert ( + salt.utils.data.encode(data, encoding, keep=True, preserve_tuples=True) == data + ) + pytest.raises( + UnicodeEncodeError, + salt.utils.data.encode, + data, + encoding, + keep=False, + preserve_tuples=True, + ) + + for index, _ in enumerate(data): + assert ( + salt.utils.data.encode( + data[index], encoding, keep=True, preserve_tuples=True + ) + == data[index] + ) + pytest.raises( + UnicodeEncodeError, + salt.utils.data.encode, + data[index], + encoding, + keep=False, + preserve_tuples=True, + ) + + +def test_encode_fallback(): + """ + Test fallback to utf-8 + """ + with patch.object(builtins, "__salt_system_encoding__", "ascii"): + assert salt.utils.data.encode("яйца") == _b("яйца") + with patch.object(builtins, "__salt_system_encoding__", "CP1252"): + assert salt.utils.data.encode("Ψ") == _b("Ψ") + + +def test_repack_dict(): + list_of_one_element_dicts = [ + {"dict_key_1": "dict_val_1"}, + {"dict_key_2": "dict_val_2"}, + {"dict_key_3": "dict_val_3"}, + ] + expected_ret = { + "dict_key_1": "dict_val_1", + "dict_key_2": "dict_val_2", + "dict_key_3": "dict_val_3", + } + ret = salt.utils.data.repack_dictlist(list_of_one_element_dicts) + assert ret == expected_ret + + # Try with yaml + yaml_key_val_pair = "- key1: val1" + ret = salt.utils.data.repack_dictlist(yaml_key_val_pair) + assert ret == {"key1": "val1"} + + # Make sure we handle non-yaml junk data + ret = salt.utils.data.repack_dictlist(LOREM_IPSUM) + assert ret == {} + + +def test_stringify(): + pytest.raises(TypeError, salt.utils.data.stringify, 9) + assert salt.utils.data.stringify(["one", "two", "three", 4, 5]) == [ + "one", "two", "three", - "one", + "4", + "5", ] + + +def test_json_query(): + # Raises exception if jmespath module is not found + with patch("salt.utils.data.jmespath", None): + with pytest.raises(RuntimeError, match="requires jmespath"): + salt.utils.data.json_query({}, "@") + + # Test search + user_groups = { + "user1": {"groups": ["group1", "group2", "group3"]}, + "user2": {"groups": ["group1", "group2"]}, + "user3": {"groups": ["group3"]}, + } + expression = "*.groups[0]" + primary_groups = ["group1", "group1", "group3"] + assert sorted(salt.utils.data.json_query(user_groups, expression)) == primary_groups + + +def test_nop(): + """ + Test cases where nothing will be done. + """ + # Test with dictionary without recursion + old_dict = { + "foo": "bar", + "bar": {"baz": {"qux": "quux"}}, + "baz": ["qux", {"foo": "bar"}], + } + new_dict = salt.utils.data.filter_falsey(old_dict) + assert old_dict == new_dict + # Check returned type equality + assert type(old_dict) is type(new_dict) + # Test dictionary with recursion + new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3) + assert old_dict == new_dict + # Test with list + old_list = ["foo", "bar"] + new_list = salt.utils.data.filter_falsey(old_list) + assert old_list == new_list + # Check returned type equality + assert type(old_list) is type(new_list) + # Test with set + old_set = {"foo", "bar"} + new_set = salt.utils.data.filter_falsey(old_set) + assert old_set == new_set + # Check returned type equality + assert type(old_set) is type(new_set) + # Test with SaltOrderedDict + old_dict = SaltOrderedDict( + [ + ("foo", "bar"), + ("bar", SaltOrderedDict([("qux", "quux")])), + ("baz", ["qux", SaltOrderedDict([("foo", "bar")])]), + ] + ) + new_dict = salt.utils.data.filter_falsey(old_dict) + assert old_dict == new_dict + assert type(old_dict) is type(new_dict) + # Test excluding int + old_list = [0] + new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[int]) + assert old_list == new_list + # Test excluding str (or unicode) (or both) + old_list = [""] + new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[str]) + assert old_list == new_list + # Test excluding list + old_list = [[]] + new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type([])]) + assert old_list == new_list + # Test excluding dict + old_list = [{}] + new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type({})]) + assert old_list == new_list + + +def test_filter_dict_no_recurse(): + """ + Test filtering a dictionary without recursing. + This will only filter out key-values where the values are falsey. + """ + old_dict = { + "foo": None, + "bar": {"baz": {"qux": None, "quux": "", "foo": []}}, + "baz": ["qux"], + "qux": {}, + "quux": [], + } + new_dict = salt.utils.data.filter_falsey(old_dict) + expect_dict = { + "bar": {"baz": {"qux": None, "quux": "", "foo": []}}, + "baz": ["qux"], + } + assert expect_dict == new_dict + assert type(expect_dict) is type(new_dict) + + +def test_filter_dict_recurse(): + """ + Test filtering a dictionary with recursing. + This will filter out any key-values where the values are falsey or when + the values *become* falsey after filtering their contents (in case they + are lists or dicts). + """ + old_dict = { + "foo": None, + "bar": {"baz": {"qux": None, "quux": "", "foo": []}}, + "baz": ["qux"], + "qux": {}, + "quux": [], + } + new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3) + expect_dict = {"baz": ["qux"]} + assert expect_dict == new_dict + assert type(expect_dict) is type(new_dict) + + +def test_filter_list_no_recurse(): + """ + Test filtering a list without recursing. + This will only filter out items which are falsey. + """ + old_list = ["foo", None, [], {}, 0, ""] + new_list = salt.utils.data.filter_falsey(old_list) + expect_list = ["foo"] + assert expect_list == new_list + assert type(expect_list) is type(new_list) + # Ensure nested values are *not* filtered out. + old_list = [ + "foo", + ["foo"], + ["foo", None], + {"foo": 0}, + {"foo": "bar", "baz": []}, + [{"foo": ""}], + ] + new_list = salt.utils.data.filter_falsey(old_list) + assert old_list == new_list + assert type(old_list) is type(new_list) + + +def test_filter_list_recurse(): + """ + Test filtering a list with recursing. + This will filter out any items which are falsey, or which become falsey + after filtering their contents (in case they are lists or dicts). + """ + old_list = [ + "foo", + ["foo"], + ["foo", None], + {"foo": 0}, + {"foo": "bar", "baz": []}, + [{"foo": ""}], + ] + new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3) + expect_list = ["foo", ["foo"], ["foo"], {"foo": "bar"}] + assert expect_list == new_list + assert type(expect_list) is type(new_list) + + +def test_filter_set_no_recurse(): + """ + Test filtering a set without recursing. + Note that a set cannot contain unhashable types, so recursion is not possible. + """ + old_set = {"foo", None, 0, ""} + new_set = salt.utils.data.filter_falsey(old_set) + expect_set = {"foo"} + assert expect_set == new_set + assert type(expect_set) is type(new_set) + + +def test_filter_ordereddict_no_recurse(): + """ + Test filtering an SaltOrderedDict without recursing. + """ + old_dict = SaltOrderedDict( + [ + ("foo", None), + ( + "bar", + SaltOrderedDict( + [ + ( + "baz", + SaltOrderedDict([("qux", None), ("quux", ""), ("foo", [])]), + ) + ] + ), + ), + ("baz", ["qux"]), + ("qux", {}), + ("quux", []), + ] + ) + new_dict = salt.utils.data.filter_falsey(old_dict) + expect_dict = SaltOrderedDict( + [ + ( + "bar", + SaltOrderedDict( + [ + ( + "baz", + SaltOrderedDict([("qux", None), ("quux", ""), ("foo", [])]), + ) + ] + ), + ), + ("baz", ["qux"]), + ] + ) + assert expect_dict == new_dict + assert type(expect_dict) is type(new_dict) + + +def test_filter_ordereddict_recurse(): + """ + Test filtering an SaltOrderedDict with recursing. + """ + old_dict = SaltOrderedDict( + [ + ("foo", None), + ( + "bar", + SaltOrderedDict( + [ + ( + "baz", + SaltOrderedDict([("qux", None), ("quux", ""), ("foo", [])]), + ) + ] + ), + ), + ("baz", ["qux"]), + ("qux", {}), + ("quux", []), + ] + ) + new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3) + expect_dict = SaltOrderedDict([("baz", ["qux"])]) + assert expect_dict == new_dict + assert type(expect_dict) is type(new_dict) + + +def test_filter_list_recurse_limit(): + """ + Test filtering a list with recursing, but with a limited depth. + Note that the top-level is always processed, so a recursion depth of 2 + means that two *additional* levels are processed. + """ + old_list = [None, [None, [None, [None]]]] + new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=2) + assert [[[[None]]]] == new_list + + +def test_filter_dict_recurse_limit(): + """ + Test filtering a dict with recursing, but with a limited depth. + Note that the top-level is always processed, so a recursion depth of 2 + means that two *additional* levels are processed. + """ + old_dict = { + "one": None, + "foo": {"two": None, "bar": {"three": None, "baz": {"four": None}}}, + } + new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=2) + assert {"foo": {"bar": {"baz": {"four": None}}}} == new_dict + + +def test_filter_exclude_types(): + """ + Test filtering a list recursively, but also ignoring (i.e. not filtering) + out certain types that can be falsey. + """ + # Ignore int, unicode + old_list = [ + "foo", + ["foo"], + ["foo", None], + {"foo": 0}, + {"foo": "bar", "baz": []}, + [{"foo": ""}], + ] + new_list = salt.utils.data.filter_falsey( + old_list, recurse_depth=3, ignore_types=[int, str] + ) + assert [ + "foo", + ["foo"], + ["foo"], + {"foo": 0}, + {"foo": "bar"}, + [{"foo": ""}], + ] == new_list + # Ignore list + old_list = [ + "foo", + ["foo"], + ["foo", None], + {"foo": 0}, + {"foo": "bar", "baz": []}, + [{"foo": ""}], + ] + new_list = salt.utils.data.filter_falsey( + old_list, recurse_depth=3, ignore_types=[type([])] + ) + assert ["foo", ["foo"], ["foo"], {"foo": "bar", "baz": []}, []] == new_list + # Ignore dict + old_list = [ + "foo", + ["foo"], + ["foo", None], + {"foo": 0}, + {"foo": "bar", "baz": []}, + [{"foo": ""}], + ] + new_list = salt.utils.data.filter_falsey( + old_list, recurse_depth=3, ignore_types=[type({})] + ) + assert ["foo", ["foo"], ["foo"], {}, {"foo": "bar"}, [{}]] == new_list + # Ignore NoneType + old_list = [ + "foo", + ["foo"], + ["foo", None], + {"foo": 0}, + {"foo": "bar", "baz": []}, + [{"foo": ""}], + ] + new_list = salt.utils.data.filter_falsey( + old_list, recurse_depth=3, ignore_types=[type(None)] + ) + assert ["foo", ["foo"], ["foo", None], {"foo": "bar"}] == new_list + + +def test_list_equality(): + """ + Test cases where equal lists are compared. + """ + test_list = [0, 1, 2] + assert {} == salt.utils.data.recursive_diff(test_list, test_list) + + test_list = [[0], [1], [0, 1, 2]] + assert {} == salt.utils.data.recursive_diff(test_list, test_list) + + +def test_dict_equality(): + """ + Test cases where equal dicts are compared. + """ + test_dict = {"foo": "bar", "bar": {"baz": {"qux": "quux"}}, "frop": 0} + assert {} == salt.utils.data.recursive_diff(test_dict, test_dict) + + +def test_ordereddict_equality(): + """ + Test cases where equal SaltOrderedDicts are compared. + """ + test_dict = SaltOrderedDict( + [ + ("foo", "bar"), + ("bar", SaltOrderedDict([("baz", SaltOrderedDict([("qux", "quux")]))])), + ("frop", 0), + ] + ) + assert {} == salt.utils.data.recursive_diff(test_dict, test_dict) + + +def test_mixed_equality(): + """ + Test cases where mixed nested lists and dicts are compared. + """ + test_data = { + "foo": "bar", + "baz": [0, 1, 2], + "bar": {"baz": [{"qux": "quux"}, {"froop", 0}]}, + } + assert {} == salt.utils.data.recursive_diff(test_data, test_data) + + +def test_set_equality(): + """ + Test cases where equal sets are compared. + """ + test_set = {0, 1, 2, 3, "foo"} + assert {} == salt.utils.data.recursive_diff(test_set, test_set) + + # This is a bit of an oddity, as python seems to sort the sets in memory + # so both sets end up with the same ordering (0..3). + set_one = {0, 1, 2, 3} + set_two = {3, 2, 1, 0} + assert {} == salt.utils.data.recursive_diff(set_one, set_two) + + +def test_tuple_equality(): + """ + Test cases where equal tuples are compared. + """ + test_tuple = (0, 1, 2, 3, "foo") + assert {} == salt.utils.data.recursive_diff(test_tuple, test_tuple) + + +def test_list_inequality(): + """ + Test cases where two inequal lists are compared. + """ + list_one = [0, 1, 2] + list_two = ["foo", "bar", "baz"] + expected_result = {"old": list_one, "new": list_two} + assert expected_result == salt.utils.data.recursive_diff(list_one, list_two) + expected_result = {"new": list_one, "old": list_two} + assert expected_result == salt.utils.data.recursive_diff(list_two, list_one) + + list_one = [0, "foo", 1, "bar"] + list_two = [1, "foo", 1, "qux"] + expected_result = {"old": [0, "bar"], "new": [1, "qux"]} + assert expected_result == salt.utils.data.recursive_diff(list_one, list_two) + expected_result = {"new": [0, "bar"], "old": [1, "qux"]} + assert expected_result == salt.utils.data.recursive_diff(list_two, list_one) + + list_one = [0, 1, [2, 3]] + list_two = [0, 1, ["foo", "bar"]] + expected_result = {"old": [[2, 3]], "new": [["foo", "bar"]]} + assert expected_result == salt.utils.data.recursive_diff(list_one, list_two) + expected_result = {"new": [[2, 3]], "old": [["foo", "bar"]]} + assert expected_result == salt.utils.data.recursive_diff(list_two, list_one) + + +def test_dict_inequality(): + """ + Test cases where two inequal dicts are compared. + """ + dict_one = {"foo": 1, "bar": 2, "baz": 3} + dict_two = {"foo": 2, 1: "bar", "baz": 3} + expected_result = {"old": {"foo": 1, "bar": 2}, "new": {"foo": 2, 1: "bar"}} + assert expected_result == salt.utils.data.recursive_diff(dict_one, dict_two) + expected_result = {"new": {"foo": 1, "bar": 2}, "old": {"foo": 2, 1: "bar"}} + assert expected_result == salt.utils.data.recursive_diff(dict_two, dict_one) + + dict_one = {"foo": {"bar": {"baz": 1}}} + dict_two = {"foo": {"qux": {"baz": 1}}} + expected_result = {"old": dict_one, "new": dict_two} + assert expected_result == salt.utils.data.recursive_diff(dict_one, dict_two) + expected_result = {"new": dict_one, "old": dict_two} + assert expected_result == salt.utils.data.recursive_diff(dict_two, dict_one) + + +def test_ordereddict_inequality(): + """ + Test cases where two inequal SaltOrderedDicts are compared. + """ + odict_one = SaltOrderedDict([("foo", "bar"), ("bar", "baz")]) + odict_two = SaltOrderedDict([("bar", "baz"), ("foo", "bar")]) + expected_result = {"old": odict_one, "new": odict_two} + assert expected_result == salt.utils.data.recursive_diff(odict_one, odict_two) + + +def test_set_inequality(): + """ + Test cases where two inequal sets are compared. + Tricky as the sets are compared zipped, so shuffled sets of equal values + are considered different. + """ + set_one = {0, 1, 2, 4} + set_two = {0, 1, 3, 4} + expected_result = {"old": {2}, "new": {3}} + assert expected_result == salt.utils.data.recursive_diff(set_one, set_two) + expected_result = {"new": {2}, "old": {3}} + assert expected_result == salt.utils.data.recursive_diff(set_two, set_one) + + # It is unknown how different python versions will store sets in memory. + # Python 2.7 seems to sort it (i.e. set_one below becomes {0, 1, 'foo', 'bar'} + # However Python 3.6.8 stores it differently each run. + # So just test for "not equal" here. + set_one = {0, "foo", 1, "bar"} + set_two = {"foo", 1, "bar", 2} + expected_result = {} + assert expected_result != salt.utils.data.recursive_diff(set_one, set_two) + + +def test_mixed_inequality(): + """ + Test cases where two mixed dicts/iterables that are different are compared. + """ + dict_one = {"foo": [1, 2, 3]} + dict_two = {"foo": [3, 2, 1]} + expected_result = {"old": {"foo": [1, 3]}, "new": {"foo": [3, 1]}} + assert expected_result == salt.utils.data.recursive_diff(dict_one, dict_two) + expected_result = {"new": {"foo": [1, 3]}, "old": {"foo": [3, 1]}} + assert expected_result == salt.utils.data.recursive_diff(dict_two, dict_one) + + list_one = [1, 2, {"foo": ["bar", {"foo": 1, "bar": 2}]}] + list_two = [3, 4, {"foo": ["qux", {"foo": 1, "bar": 2}]}] + expected_result = { + "old": [1, 2, {"foo": ["bar"]}], + "new": [3, 4, {"foo": ["qux"]}], + } + assert expected_result == salt.utils.data.recursive_diff(list_one, list_two) + expected_result = { + "new": [1, 2, {"foo": ["bar"]}], + "old": [3, 4, {"foo": ["qux"]}], + } + assert expected_result == salt.utils.data.recursive_diff(list_two, list_one) + + mixed_one = {"foo": {0, 1, 2}, "bar": [0, 1, 2]} + mixed_two = {"foo": {1, 2, 3}, "bar": [1, 2, 3]} + expected_result = { + "old": {"foo": {0}, "bar": [0, 1, 2]}, + "new": {"foo": {3}, "bar": [1, 2, 3]}, + } + assert expected_result == salt.utils.data.recursive_diff(mixed_one, mixed_two) + expected_result = { + "new": {"foo": {0}, "bar": [0, 1, 2]}, + "old": {"foo": {3}, "bar": [1, 2, 3]}, + } + assert expected_result == salt.utils.data.recursive_diff(mixed_two, mixed_one) + + +def test_tuple_inequality(): + """ + Test cases where two tuples that are different are compared. + """ + tuple_one = (1, 2, 3) + tuple_two = (3, 2, 1) + expected_result = {"old": (1, 3), "new": (3, 1)} + assert expected_result == salt.utils.data.recursive_diff(tuple_one, tuple_two) + + +def test_list_vs_set(): + """ + Test case comparing a list with a set, will be compared unordered. + """ + mixed_one = [1, 2, 3] + mixed_two = {3, 2, 1} + expected_result = {} + assert expected_result == salt.utils.data.recursive_diff(mixed_one, mixed_two) + assert expected_result == salt.utils.data.recursive_diff(mixed_two, mixed_one) + + +def test_dict_vs_ordereddict(): + """ + Test case comparing a dict with an ordereddict, will be compared unordered. + """ + test_dict = {"foo": "bar", "bar": "baz"} + test_odict = SaltOrderedDict([("foo", "bar"), ("bar", "baz")]) + assert {} == salt.utils.data.recursive_diff(test_dict, test_odict) + assert {} == salt.utils.data.recursive_diff(test_odict, test_dict) + + test_odict2 = SaltOrderedDict([("bar", "baz"), ("foo", "bar")]) + assert {} == salt.utils.data.recursive_diff(test_dict, test_odict2) + assert {} == salt.utils.data.recursive_diff(test_odict2, test_dict) + + +def test_list_ignore_ignored(): + """ + Test case comparing two lists with ignore-list supplied (which is not used + when comparing lists). + """ + list_one = [1, 2, 3] + list_two = [3, 2, 1] + expected_result = {"old": [1, 3], "new": [3, 1]} + assert expected_result == salt.utils.data.recursive_diff( + list_one, list_two, ignore_keys=[1, 3] + ) + + +def test_dict_ignore(): + """ + Test case comparing two dicts with ignore-list supplied. + """ + dict_one = {"foo": 1, "bar": 2, "baz": 3} + dict_two = {"foo": 3, "bar": 2, "baz": 1} + expected_result = {"old": {"baz": 3}, "new": {"baz": 1}} + assert expected_result == salt.utils.data.recursive_diff( + dict_one, dict_two, ignore_keys=["foo"] + ) + + +def test_ordereddict_ignore(): + """ + Test case comparing two SaltOrderedDicts with ignore-list supplied. + """ + odict_one = SaltOrderedDict([("foo", 1), ("bar", 2), ("baz", 3)]) + odict_two = SaltOrderedDict([("baz", 1), ("bar", 2), ("foo", 3)]) + # The key 'foo' will be ignored, which means the key from the other SaltOrderedDict + # will always be considered "different" since SaltOrderedDicts are compared ordered. + expected_result = { + "old": SaltOrderedDict([("baz", 3)]), + "new": SaltOrderedDict([("baz", 1)]), + } + assert expected_result == salt.utils.data.recursive_diff( + odict_one, odict_two, ignore_keys=["foo"] + ) + + +def test_dict_vs_ordereddict_ignore(): + """ + Test case comparing a dict with an SaltOrderedDict with ignore-list supplied. + """ + dict_one = {"foo": 1, "bar": 2, "baz": 3} + odict_two = SaltOrderedDict([("foo", 3), ("bar", 2), ("baz", 1)]) + expected_result = {"old": {"baz": 3}, "new": SaltOrderedDict([("baz", 1)])} + assert expected_result == salt.utils.data.recursive_diff( + dict_one, odict_two, ignore_keys=["foo"] + ) + + +def test_mixed_nested_ignore(): + """ + Test case comparing mixed, nested items with ignore-list supplied. + """ + dict_one = {"foo": [1], "bar": {"foo": 1, "bar": 2}, "baz": 3} + dict_two = {"foo": [2], "bar": {"foo": 3, "bar": 2}, "baz": 1} + expected_result = {"old": {"baz": 3}, "new": {"baz": 1}} + assert expected_result == salt.utils.data.recursive_diff( + dict_one, dict_two, ignore_keys=["foo"] + ) + + +def test_ordered_dict_unequal_length(): + """ + Test case comparing two SaltOrderedDicts of unequal length. + """ + odict_one = SaltOrderedDict([("foo", 1), ("bar", 2), ("baz", 3)]) + odict_two = SaltOrderedDict([("foo", 1), ("bar", 2)]) + expected_result = {"old": SaltOrderedDict([("baz", 3)]), "new": {}} + assert expected_result == salt.utils.data.recursive_diff(odict_one, odict_two) + + +def test_list_unequal_length(): + """ + Test case comparing two lists of unequal length. + """ + list_one = [1, 2, 3] + list_two = [1, 2, 3, 4] + expected_result = {"old": [], "new": [4]} + assert expected_result == salt.utils.data.recursive_diff(list_one, list_two) + + +def test_set_unequal_length(): + """ + Test case comparing two sets of unequal length. + This does not do anything special, as it is unordered. + """ + set_one = {1, 2, 3} + set_two = {4, 3, 2, 1} + expected_result = {"old": set(), "new": {4}} + assert expected_result == salt.utils.data.recursive_diff(set_one, set_two) + + +def test_tuple_unequal_length(): + """ + Test case comparing two tuples of unequal length. + This should be the same as comparing two ordered lists. + """ + tuple_one = (1, 2, 3) + tuple_two = (1, 2, 3, 4) + expected_result = {"old": (), "new": (4,)} + assert expected_result == salt.utils.data.recursive_diff(tuple_one, tuple_two) + + +def test_list_unordered(): + """ + Test case comparing two lists unordered. + """ + list_one = [1, 2, 3, 4] + list_two = [4, 3, 2] + expected_result = {"old": [1], "new": []} + assert expected_result == salt.utils.data.recursive_diff( + list_one, list_two, ignore_order=True + ) + + +def test_mixed_nested_unordered(): + """ + Test case comparing nested dicts/lists unordered. + """ + dict_one = {"foo": {"bar": [1, 2, 3]}, "bar": [{"foo": 4}, 0]} + dict_two = {"foo": {"bar": [3, 2, 1]}, "bar": [0, {"foo": 4}]} + expected_result = {} + assert expected_result == salt.utils.data.recursive_diff( + dict_one, dict_two, ignore_order=True + ) + expected_result = { + "old": {"foo": {"bar": [1, 3]}, "bar": [{"foo": 4}, 0]}, + "new": {"foo": {"bar": [3, 1]}, "bar": [0, {"foo": 4}]}, + } + assert expected_result == salt.utils.data.recursive_diff(dict_one, dict_two) + + +def test_ordered_dict_unordered(): + """ + Test case comparing SaltOrderedDicts unordered. + """ + odict_one = SaltOrderedDict([("foo", 1), ("bar", 2), ("baz", 3)]) + odict_two = SaltOrderedDict([("baz", 3), ("bar", 2), ("foo", 1)]) + expected_result = {} + assert expected_result == salt.utils.data.recursive_diff( + odict_one, odict_two, ignore_order=True + ) + + +def test_ignore_missing_keys_dict(): + """ + Test case ignoring missing keys on a comparison of dicts. + """ + dict_one = {"foo": 1, "bar": 2, "baz": 3} + dict_two = {"bar": 3} + expected_result = {"old": {"bar": 2}, "new": {"bar": 3}} + assert expected_result == salt.utils.data.recursive_diff( + dict_one, dict_two, ignore_missing_keys=True + ) + + +def test_ignore_missing_keys_ordered_dict(): + """ + Test case not ignoring missing keys on a comparison of SaltOrderedDicts. + """ + odict_one = SaltOrderedDict([("foo", 1), ("bar", 2), ("baz", 3)]) + odict_two = SaltOrderedDict([("bar", 3)]) + expected_result = {"old": odict_one, "new": odict_two} + assert expected_result == salt.utils.data.recursive_diff( + odict_one, odict_two, ignore_missing_keys=True + ) + + +def test_ignore_missing_keys_recursive(): + """ + Test case ignoring missing keys on a comparison of nested dicts. + """ + dict_one = {"foo": {"bar": 2, "baz": 3}} + dict_two = {"foo": {"baz": 3}} + expected_result = {} + assert expected_result == salt.utils.data.recursive_diff( + dict_one, dict_two, ignore_missing_keys=True + ) + # Compare from dict-in-dict + dict_two = {} + assert expected_result == salt.utils.data.recursive_diff( + dict_one, dict_two, ignore_missing_keys=True + ) + # Compare from dict-in-list + dict_one = {"foo": ["bar", {"baz": 3}]} + dict_two = {"foo": ["bar", {}]} + assert expected_result == salt.utils.data.recursive_diff( + dict_one, dict_two, ignore_missing_keys=True + ) + + +def test_ordereddict_dict_items(): + """ + Test case for SaltOrderedDicts items. + """ + dict_one = {"foo": 1, "bar": 2, "baz": 3} + expected_items = 'odict_items([("foo", 1), ("bar", 2), ("baz", 3)])' + + odict_one = SaltOrderedDict(dict_one) + myodict = odict_one.items() + log.warning(f"DGM test_ordered_dict_items items '{str(myodict)}'") + ## assert odict_one.items() == expected_items + + myodict = odict_one.keys() + log.warning(f"DGM test_ordered_dict_items keys '{myodict}'") + ## assert odict_one.items() == expected + + myodict = odict_one.values() + log.warning(f"DGM test_ordered_dict_items values '{myodict}'") + ## assert odict_one.items() == expected + + myodict = odict_one.popitem() + log.warning(f"DGM test_ordered_dict_items popitem '{myodict}'") + ## assert odict_one.items() == expected + + odict_one.clear() + myodict = odict_one.popitem() + log.warning(f"DGM test_ordered_dict_items clear '{myodict}'") + ## assert odict_one.items() == expected + + dgm_dir = dir(odict_one) + log.warning(f"DGM test_ordered_dict_items dgm_dir iterkeys '{dgm_dir}'") + + for myodict in odict_one.iterkeys(): + log.warning(f"DGM test_ordered_dict_items iterkeys '{myodict}'") + ## assert odict_one.items() == expected + + for myodict in odict_one.itervalues(): + log.warning(f"DGM test_ordered_dict_items itervalues '{myodict}'") + ## assert odict_one.items() == expected diff --git a/tests/unit/utils/test_data.py b/tests/unit/utils/test_data.py deleted file mode 100644 index 6d16e198999..00000000000 --- a/tests/unit/utils/test_data.py +++ /dev/null @@ -1,1506 +0,0 @@ -""" -Tests for salt.utils.data -""" - -import builtins -import logging - -import salt.utils.data -import salt.utils.stringutils -from salt.utils.odict import OrderedDict -from tests.support.mock import patch -from tests.support.unit import LOREM_IPSUM, TestCase - -log = logging.getLogger(__name__) -_b = lambda x: x.encode("utf-8") -_s = lambda x: salt.utils.stringutils.to_str(x, normalize=True) -# Some randomized data that will not decode -BYTES = b"1\x814\x10" - -# This is an example of a unicode string with й constructed using two separate -# code points. Do not modify it. -EGGS = "\u044f\u0438\u0306\u0446\u0430" - - -class DataTestCase(TestCase): - test_data = [ - "unicode_str", - _b("питон"), - 123, - 456.789, - True, - False, - None, - EGGS, - BYTES, - [123, 456.789, _b("спам"), True, False, None, EGGS, BYTES], - (987, 654.321, _b("яйца"), EGGS, None, (True, EGGS, BYTES)), - { - _b("str_key"): _b("str_val"), - None: True, - 123: 456.789, - EGGS: BYTES, - _b("subdict"): { - "unicode_key": EGGS, - _b("tuple"): (123, "hello", _b("world"), True, EGGS, BYTES), - _b("list"): [456, _b("спам"), False, EGGS, BYTES], - }, - }, - OrderedDict([(_b("foo"), "bar"), (123, 456), (EGGS, BYTES)]), - ] - - def test_sorted_ignorecase(self): - test_list = ["foo", "Foo", "bar", "Bar"] - expected_list = ["bar", "Bar", "foo", "Foo"] - self.assertEqual(salt.utils.data.sorted_ignorecase(test_list), expected_list) - - def test_mysql_to_dict(self): - test_mysql_output = [ - "+----+------+-----------+------+---------+------+-------+------------------+", - "| Id | User | Host | db | Command | Time | State | Info " - " |", - "+----+------+-----------+------+---------+------+-------+------------------+", - "| 7 | root | localhost | NULL | Query | 0 | init | show" - " processlist |", - "+----+------+-----------+------+---------+------+-------+------------------+", - ] - - ret = salt.utils.data.mysql_to_dict(test_mysql_output, "Info") - expected_dict = { - "show processlist": { - "Info": "show processlist", - "db": "NULL", - "State": "init", - "Host": "localhost", - "Command": "Query", - "User": "root", - "Time": 0, - "Id": 7, - } - } - - self.assertDictEqual(ret, expected_dict) - - def test_subdict_match(self): - test_two_level_dict = {"foo": {"bar": "baz"}} - test_two_level_comb_dict = {"foo": {"bar": "baz:woz"}} - test_two_level_dict_and_list = { - "abc": ["def", "ghi", {"lorem": {"ipsum": [{"dolor": "sit"}]}}], - } - test_three_level_dict = {"a": {"b": {"c": "v"}}} - - self.assertTrue( - salt.utils.data.subdict_match(test_two_level_dict, "foo:bar:baz") - ) - # In test_two_level_comb_dict, 'foo:bar' corresponds to 'baz:woz', not - # 'baz'. This match should return False. - self.assertFalse( - salt.utils.data.subdict_match(test_two_level_comb_dict, "foo:bar:baz") - ) - # This tests matching with the delimiter in the value part (in other - # words, that the path 'foo:bar' corresponds to the string 'baz:woz'). - self.assertTrue( - salt.utils.data.subdict_match(test_two_level_comb_dict, "foo:bar:baz:woz") - ) - # This would match if test_two_level_comb_dict['foo']['bar'] was equal - # to 'baz:woz:wiz', or if there was more deep nesting. But it does not, - # so this should return False. - self.assertFalse( - salt.utils.data.subdict_match( - test_two_level_comb_dict, "foo:bar:baz:woz:wiz" - ) - ) - # This tests for cases when a key path corresponds to a list. The - # value part 'ghi' should be successfully matched as it is a member of - # the list corresponding to key path 'abc'. It is somewhat a - # duplication of a test within test_traverse_dict_and_list, but - # salt.utils.data.subdict_match() does more than just invoke - # salt.utils.traverse_list_and_dict() so this particular assertion is a - # sanity check. - self.assertTrue( - salt.utils.data.subdict_match(test_two_level_dict_and_list, "abc:ghi") - ) - # This tests the use case of a dict embedded in a list, embedded in a - # list, embedded in a dict. This is a rather absurd case, but it - # confirms that match recursion works properly. - self.assertTrue( - salt.utils.data.subdict_match( - test_two_level_dict_and_list, "abc:lorem:ipsum:dolor:sit" - ) - ) - # Test four level dict match for reference - self.assertTrue(salt.utils.data.subdict_match(test_three_level_dict, "a:b:c:v")) - # Test regression in 2015.8 where 'a:c:v' would match 'a:b:c:v' - self.assertFalse(salt.utils.data.subdict_match(test_three_level_dict, "a:c:v")) - # Test wildcard match - self.assertTrue(salt.utils.data.subdict_match(test_three_level_dict, "a:*:c:v")) - - def test_subdict_match_with_wildcards(self): - """ - Tests subdict matching when wildcards are used in the expression - """ - data = {"a": {"b": {"ç": "d", "é": ["eff", "gee", "8ch"], "ĩ": {"j": "k"}}}} - assert salt.utils.data.subdict_match(data, "*:*:*:*") - assert salt.utils.data.subdict_match(data, "a:*:*:*") - assert salt.utils.data.subdict_match(data, "a:b:*:*") - assert salt.utils.data.subdict_match(data, "a:b:ç:*") - assert salt.utils.data.subdict_match(data, "a:b:*:d") - assert salt.utils.data.subdict_match(data, "a:*:ç:d") - assert salt.utils.data.subdict_match(data, "*:b:ç:d") - assert salt.utils.data.subdict_match(data, "*:*:ç:d") - assert salt.utils.data.subdict_match(data, "*:*:*:d") - assert salt.utils.data.subdict_match(data, "a:*:*:d") - assert salt.utils.data.subdict_match(data, "a:b:*:ef*") - assert salt.utils.data.subdict_match(data, "a:b:*:g*") - assert salt.utils.data.subdict_match(data, "a:b:*:j:*") - assert salt.utils.data.subdict_match(data, "a:b:*:j:k") - assert salt.utils.data.subdict_match(data, "a:b:*:*:k") - assert salt.utils.data.subdict_match(data, "a:b:*:*:*") - - def test_traverse_dict(self): - test_two_level_dict = {"foo": {"bar": "baz"}} - - self.assertDictEqual( - {"not_found": "nope"}, - salt.utils.data.traverse_dict( - test_two_level_dict, "foo:bar:baz", {"not_found": "nope"} - ), - ) - self.assertEqual( - "baz", - salt.utils.data.traverse_dict( - test_two_level_dict, "foo:bar", {"not_found": "not_found"} - ), - ) - - def test_traverse_dict_and_list(self): - test_two_level_dict = {"foo": {"bar": "baz"}} - test_two_level_dict_and_list = { - "foo": ["bar", "baz", {"lorem": {"ipsum": [{"dolor": "sit"}]}}] - } - - # Check traversing too far: salt.utils.data.traverse_dict_and_list() returns - # the value corresponding to a given key path, and baz is a value - # corresponding to the key path foo:bar. - self.assertDictEqual( - {"not_found": "nope"}, - salt.utils.data.traverse_dict_and_list( - test_two_level_dict, "foo:bar:baz", {"not_found": "nope"} - ), - ) - # Now check to ensure that foo:bar corresponds to baz - self.assertEqual( - "baz", - salt.utils.data.traverse_dict_and_list( - test_two_level_dict, "foo:bar", {"not_found": "not_found"} - ), - ) - # Check traversing too far - self.assertDictEqual( - {"not_found": "nope"}, - salt.utils.data.traverse_dict_and_list( - test_two_level_dict_and_list, "foo:bar", {"not_found": "nope"} - ), - ) - # Check index 1 (2nd element) of list corresponding to path 'foo' - self.assertEqual( - "baz", - salt.utils.data.traverse_dict_and_list( - test_two_level_dict_and_list, "foo:1", {"not_found": "not_found"} - ), - ) - # Traverse a couple times into dicts embedded in lists - self.assertEqual( - "sit", - salt.utils.data.traverse_dict_and_list( - test_two_level_dict_and_list, - "foo:lorem:ipsum:dolor", - {"not_found": "not_found"}, - ), - ) - - # Traverse and match integer key in a nested dict - # https://github.com/saltstack/salt/issues/56444 - self.assertEqual( - "it worked", - salt.utils.data.traverse_dict_and_list( - {"foo": {1234: "it worked"}}, - "foo:1234", - "it didn't work", - ), - ) - # Make sure that we properly return the default value when the initial - # attempt fails and YAML-loading the target key doesn't change its - # value. - self.assertEqual( - "default", - salt.utils.data.traverse_dict_and_list( - {"foo": {"baz": "didn't work"}}, - "foo:bar", - "default", - ), - ) - - def test_issue_39709(self): - test_two_level_dict_and_list = { - "foo": ["bar", "baz", {"lorem": {"ipsum": [{"dolor": "sit"}]}}] - } - - self.assertEqual( - "sit", - salt.utils.data.traverse_dict_and_list( - test_two_level_dict_and_list, - ["foo", "lorem", "ipsum", "dolor"], - {"not_found": "not_found"}, - ), - ) - - def test_compare_dicts(self): - ret = salt.utils.data.compare_dicts(old={"foo": "bar"}, new={"foo": "bar"}) - self.assertEqual(ret, {}) - - ret = salt.utils.data.compare_dicts(old={"foo": "bar"}, new={"foo": "woz"}) - expected_ret = {"foo": {"new": "woz", "old": "bar"}} - self.assertDictEqual(ret, expected_ret) - - def test_compare_lists_no_change(self): - ret = salt.utils.data.compare_lists( - old=[1, 2, 3, "a", "b", "c"], new=[1, 2, 3, "a", "b", "c"] - ) - expected = {} - self.assertDictEqual(ret, expected) - - def test_compare_lists_changes(self): - ret = salt.utils.data.compare_lists( - old=[1, 2, 3, "a", "b", "c"], new=[1, 2, 4, "x", "y", "z"] - ) - expected = {"new": [4, "x", "y", "z"], "old": [3, "a", "b", "c"]} - self.assertDictEqual(ret, expected) - - def test_compare_lists_changes_new(self): - ret = salt.utils.data.compare_lists(old=[1, 2, 3], new=[1, 2, 3, "x", "y", "z"]) - expected = {"new": ["x", "y", "z"]} - self.assertDictEqual(ret, expected) - - def test_compare_lists_changes_old(self): - ret = salt.utils.data.compare_lists(old=[1, 2, 3, "a", "b", "c"], new=[1, 2, 3]) - expected = {"old": ["a", "b", "c"]} - self.assertDictEqual(ret, expected) - - def test_decode(self): - """ - Companion to test_decode_to_str, they should both be kept up-to-date - with one another. - - NOTE: This uses the lambda "_b" defined above in the global scope, - which encodes a string to a bytestring, assuming utf-8. - """ - expected = [ - "unicode_str", - "питон", - 123, - 456.789, - True, - False, - None, - "яйца", - BYTES, - [123, 456.789, "спам", True, False, None, "яйца", BYTES], - (987, 654.321, "яйца", "яйца", None, (True, "яйца", BYTES)), - { - "str_key": "str_val", - None: True, - 123: 456.789, - "яйца": BYTES, - "subdict": { - "unicode_key": "яйца", - "tuple": (123, "hello", "world", True, "яйца", BYTES), - "list": [456, "спам", False, "яйца", BYTES], - }, - }, - OrderedDict([("foo", "bar"), (123, 456), ("яйца", BYTES)]), - ] - - ret = salt.utils.data.decode( - self.test_data, - keep=True, - normalize=True, - preserve_dict_class=True, - preserve_tuples=True, - ) - self.assertEqual(ret, expected) - - # The binary data in the data structure should fail to decode, even - # using the fallback, and raise an exception. - self.assertRaises( - UnicodeDecodeError, - salt.utils.data.decode, - self.test_data, - keep=False, - normalize=True, - preserve_dict_class=True, - preserve_tuples=True, - ) - - # Now munge the expected data so that we get what we would expect if we - # disable preservation of dict class and tuples - expected[10] = [987, 654.321, "яйца", "яйца", None, [True, "яйца", BYTES]] - expected[11]["subdict"]["tuple"] = [123, "hello", "world", True, "яйца", BYTES] - expected[12] = {"foo": "bar", 123: 456, "яйца": BYTES} - - ret = salt.utils.data.decode( - self.test_data, - keep=True, - normalize=True, - preserve_dict_class=False, - preserve_tuples=False, - ) - self.assertEqual(ret, expected) - - # Now test single non-string, non-data-structure items, these should - # return the same value when passed to this function - for item in (123, 4.56, True, False, None): - log.debug("Testing decode of %s", item) - self.assertEqual(salt.utils.data.decode(item), item) - - # Test single strings (not in a data structure) - self.assertEqual(salt.utils.data.decode("foo"), "foo") - self.assertEqual(salt.utils.data.decode(_b("bar")), "bar") - self.assertEqual(salt.utils.data.decode(EGGS, normalize=True), "яйца") - self.assertEqual(salt.utils.data.decode(EGGS, normalize=False), EGGS) - - # Test binary blob - self.assertEqual(salt.utils.data.decode(BYTES, keep=True), BYTES) - self.assertRaises(UnicodeDecodeError, salt.utils.data.decode, BYTES, keep=False) - - def test_circular_refs_dicts(self): - test_dict = {"key": "value", "type": "test1"} - test_dict["self"] = test_dict - ret = salt.utils.data._remove_circular_refs(ob=test_dict) - self.assertDictEqual(ret, {"key": "value", "type": "test1", "self": None}) - - def test_circular_refs_lists(self): - test_list = { - "foo": [], - } - test_list["foo"].append((test_list,)) - ret = salt.utils.data._remove_circular_refs(ob=test_list) - self.assertDictEqual(ret, {"foo": [(None,)]}) - - def test_circular_refs_tuple(self): - test_dup = {"foo": "string 1", "bar": "string 1", "ham": 1, "spam": 1} - ret = salt.utils.data._remove_circular_refs(ob=test_dup) - self.assertDictEqual( - ret, {"foo": "string 1", "bar": "string 1", "ham": 1, "spam": 1} - ) - - def test_decode_to_str(self): - """ - Companion to test_decode, they should both be kept up-to-date with one - another. - - NOTE: This uses the lambda "_s" defined above in the global scope, - which converts the string/bytestring to a str type. - """ - expected = [ - _s("unicode_str"), - _s("питон"), - 123, - 456.789, - True, - False, - None, - _s("яйца"), - BYTES, - [123, 456.789, _s("спам"), True, False, None, _s("яйца"), BYTES], - (987, 654.321, _s("яйца"), _s("яйца"), None, (True, _s("яйца"), BYTES)), - { - _s("str_key"): _s("str_val"), - None: True, - 123: 456.789, - _s("яйца"): BYTES, - _s("subdict"): { - _s("unicode_key"): _s("яйца"), - _s("tuple"): ( - 123, - _s("hello"), - _s("world"), - True, - _s("яйца"), - BYTES, - ), - _s("list"): [456, _s("спам"), False, _s("яйца"), BYTES], - }, - }, - OrderedDict([(_s("foo"), _s("bar")), (123, 456), (_s("яйца"), BYTES)]), - ] - - ret = salt.utils.data.decode( - self.test_data, - keep=True, - normalize=True, - preserve_dict_class=True, - preserve_tuples=True, - to_str=True, - ) - self.assertEqual(ret, expected) - - # The binary data in the data structure should fail to decode, even - # using the fallback, and raise an exception. - self.assertRaises( - UnicodeDecodeError, - salt.utils.data.decode, - self.test_data, - keep=False, - normalize=True, - preserve_dict_class=True, - preserve_tuples=True, - to_str=True, - ) - - # Now munge the expected data so that we get what we would expect if we - # disable preservation of dict class and tuples - expected[10] = [ - 987, - 654.321, - _s("яйца"), - _s("яйца"), - None, - [True, _s("яйца"), BYTES], - ] - expected[11][_s("subdict")][_s("tuple")] = [ - 123, - _s("hello"), - _s("world"), - True, - _s("яйца"), - BYTES, - ] - expected[12] = {_s("foo"): _s("bar"), 123: 456, _s("яйца"): BYTES} - - ret = salt.utils.data.decode( - self.test_data, - keep=True, - normalize=True, - preserve_dict_class=False, - preserve_tuples=False, - to_str=True, - ) - self.assertEqual(ret, expected) - - # Now test single non-string, non-data-structure items, these should - # return the same value when passed to this function - for item in (123, 4.56, True, False, None): - log.debug("Testing decode of %s", item) - self.assertEqual(salt.utils.data.decode(item, to_str=True), item) - - # Test single strings (not in a data structure) - self.assertEqual(salt.utils.data.decode("foo", to_str=True), _s("foo")) - self.assertEqual(salt.utils.data.decode(_b("bar"), to_str=True), _s("bar")) - - # Test binary blob - self.assertEqual(salt.utils.data.decode(BYTES, keep=True, to_str=True), BYTES) - self.assertRaises( - UnicodeDecodeError, - salt.utils.data.decode, - BYTES, - keep=False, - to_str=True, - ) - - def test_decode_fallback(self): - """ - Test fallback to utf-8 - """ - with patch.object(builtins, "__salt_system_encoding__", "ascii"): - self.assertEqual(salt.utils.data.decode(_b("яйца")), "яйца") - - def test_encode(self): - """ - NOTE: This uses the lambda "_b" defined above in the global scope, - which encodes a string to a bytestring, assuming utf-8. - """ - expected = [ - _b("unicode_str"), - _b("питон"), - 123, - 456.789, - True, - False, - None, - _b(EGGS), - BYTES, - [123, 456.789, _b("спам"), True, False, None, _b(EGGS), BYTES], - (987, 654.321, _b("яйца"), _b(EGGS), None, (True, _b(EGGS), BYTES)), - { - _b("str_key"): _b("str_val"), - None: True, - 123: 456.789, - _b(EGGS): BYTES, - _b("subdict"): { - _b("unicode_key"): _b(EGGS), - _b("tuple"): (123, _b("hello"), _b("world"), True, _b(EGGS), BYTES), - _b("list"): [456, _b("спам"), False, _b(EGGS), BYTES], - }, - }, - OrderedDict([(_b("foo"), _b("bar")), (123, 456), (_b(EGGS), BYTES)]), - ] - - # Both keep=True and keep=False should work because the BYTES data is - # already bytes. - ret = salt.utils.data.encode( - self.test_data, keep=True, preserve_dict_class=True, preserve_tuples=True - ) - self.assertEqual(ret, expected) - ret = salt.utils.data.encode( - self.test_data, keep=False, preserve_dict_class=True, preserve_tuples=True - ) - self.assertEqual(ret, expected) - - # Now munge the expected data so that we get what we would expect if we - # disable preservation of dict class and tuples - expected[10] = [ - 987, - 654.321, - _b("яйца"), - _b(EGGS), - None, - [True, _b(EGGS), BYTES], - ] - expected[11][_b("subdict")][_b("tuple")] = [ - 123, - _b("hello"), - _b("world"), - True, - _b(EGGS), - BYTES, - ] - expected[12] = {_b("foo"): _b("bar"), 123: 456, _b(EGGS): BYTES} - - ret = salt.utils.data.encode( - self.test_data, keep=True, preserve_dict_class=False, preserve_tuples=False - ) - self.assertEqual(ret, expected) - ret = salt.utils.data.encode( - self.test_data, keep=False, preserve_dict_class=False, preserve_tuples=False - ) - self.assertEqual(ret, expected) - - # Now test single non-string, non-data-structure items, these should - # return the same value when passed to this function - for item in (123, 4.56, True, False, None): - log.debug("Testing encode of %s", item) - self.assertEqual(salt.utils.data.encode(item), item) - - # Test single strings (not in a data structure) - self.assertEqual(salt.utils.data.encode("foo"), _b("foo")) - self.assertEqual(salt.utils.data.encode(_b("bar")), _b("bar")) - - # Test binary blob, nothing should happen even when keep=False since - # the data is already bytes - self.assertEqual(salt.utils.data.encode(BYTES, keep=True), BYTES) - self.assertEqual(salt.utils.data.encode(BYTES, keep=False), BYTES) - - def test_encode_keep(self): - """ - Whereas we tested the keep argument in test_decode, it is much easier - to do a more comprehensive test of keep in its own function where we - can force the encoding. - """ - unicode_str = "питон" - encoding = "ascii" - - # Test single string - self.assertEqual( - salt.utils.data.encode(unicode_str, encoding, keep=True), unicode_str - ) - self.assertRaises( - UnicodeEncodeError, - salt.utils.data.encode, - unicode_str, - encoding, - keep=False, - ) - - data = [ - unicode_str, - [b"foo", [unicode_str], {b"key": unicode_str}, (unicode_str,)], - { - b"list": [b"foo", unicode_str], - b"dict": {b"key": unicode_str}, - b"tuple": (b"foo", unicode_str), - }, - ([b"foo", unicode_str], {b"key": unicode_str}, (unicode_str,)), - ] - - # Since everything was a bytestring aside from the bogus data, the - # return data should be identical. We don't need to test recursive - # decoding, that has already been tested in test_encode. - self.assertEqual( - salt.utils.data.encode(data, encoding, keep=True, preserve_tuples=True), - data, - ) - self.assertRaises( - UnicodeEncodeError, - salt.utils.data.encode, - data, - encoding, - keep=False, - preserve_tuples=True, - ) - - for index, _ in enumerate(data): - self.assertEqual( - salt.utils.data.encode( - data[index], encoding, keep=True, preserve_tuples=True - ), - data[index], - ) - self.assertRaises( - UnicodeEncodeError, - salt.utils.data.encode, - data[index], - encoding, - keep=False, - preserve_tuples=True, - ) - - def test_encode_fallback(self): - """ - Test fallback to utf-8 - """ - with patch.object(builtins, "__salt_system_encoding__", "ascii"): - self.assertEqual(salt.utils.data.encode("яйца"), _b("яйца")) - with patch.object(builtins, "__salt_system_encoding__", "CP1252"): - self.assertEqual(salt.utils.data.encode("Ψ"), _b("Ψ")) - - def test_repack_dict(self): - list_of_one_element_dicts = [ - {"dict_key_1": "dict_val_1"}, - {"dict_key_2": "dict_val_2"}, - {"dict_key_3": "dict_val_3"}, - ] - expected_ret = { - "dict_key_1": "dict_val_1", - "dict_key_2": "dict_val_2", - "dict_key_3": "dict_val_3", - } - ret = salt.utils.data.repack_dictlist(list_of_one_element_dicts) - self.assertDictEqual(ret, expected_ret) - - # Try with yaml - yaml_key_val_pair = "- key1: val1" - ret = salt.utils.data.repack_dictlist(yaml_key_val_pair) - self.assertDictEqual(ret, {"key1": "val1"}) - - # Make sure we handle non-yaml junk data - ret = salt.utils.data.repack_dictlist(LOREM_IPSUM) - self.assertDictEqual(ret, {}) - - def test_stringify(self): - self.assertRaises(TypeError, salt.utils.data.stringify, 9) - self.assertEqual( - salt.utils.data.stringify(["one", "two", "three", 4, 5]), - ["one", "two", "three", "4", "5"], - ) - - def test_json_query(self): - # Raises exception if jmespath module is not found - with patch("salt.utils.data.jmespath", None): - self.assertRaisesRegex( - RuntimeError, "requires jmespath", salt.utils.data.json_query, {}, "@" - ) - - # Test search - user_groups = { - "user1": {"groups": ["group1", "group2", "group3"]}, - "user2": {"groups": ["group1", "group2"]}, - "user3": {"groups": ["group3"]}, - } - expression = "*.groups[0]" - primary_groups = ["group1", "group1", "group3"] - self.assertEqual( - sorted(salt.utils.data.json_query(user_groups, expression)), primary_groups - ) - - -class FilterFalseyTestCase(TestCase): - """ - Test suite for salt.utils.data.filter_falsey - """ - - def test_nop(self): - """ - Test cases where nothing will be done. - """ - # Test with dictionary without recursion - old_dict = { - "foo": "bar", - "bar": {"baz": {"qux": "quux"}}, - "baz": ["qux", {"foo": "bar"}], - } - new_dict = salt.utils.data.filter_falsey(old_dict) - self.assertEqual(old_dict, new_dict) - # Check returned type equality - self.assertIs(type(old_dict), type(new_dict)) - # Test dictionary with recursion - new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3) - self.assertEqual(old_dict, new_dict) - # Test with list - old_list = ["foo", "bar"] - new_list = salt.utils.data.filter_falsey(old_list) - self.assertEqual(old_list, new_list) - # Check returned type equality - self.assertIs(type(old_list), type(new_list)) - # Test with set - old_set = {"foo", "bar"} - new_set = salt.utils.data.filter_falsey(old_set) - self.assertEqual(old_set, new_set) - # Check returned type equality - self.assertIs(type(old_set), type(new_set)) - # Test with OrderedDict - old_dict = OrderedDict( - [ - ("foo", "bar"), - ("bar", OrderedDict([("qux", "quux")])), - ("baz", ["qux", OrderedDict([("foo", "bar")])]), - ] - ) - new_dict = salt.utils.data.filter_falsey(old_dict) - self.assertEqual(old_dict, new_dict) - self.assertIs(type(old_dict), type(new_dict)) - # Test excluding int - old_list = [0] - new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[int]) - self.assertEqual(old_list, new_list) - # Test excluding str (or unicode) (or both) - old_list = [""] - new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[str]) - self.assertEqual(old_list, new_list) - # Test excluding list - old_list = [[]] - new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type([])]) - self.assertEqual(old_list, new_list) - # Test excluding dict - old_list = [{}] - new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type({})]) - self.assertEqual(old_list, new_list) - - def test_filter_dict_no_recurse(self): - """ - Test filtering a dictionary without recursing. - This will only filter out key-values where the values are falsey. - """ - old_dict = { - "foo": None, - "bar": {"baz": {"qux": None, "quux": "", "foo": []}}, - "baz": ["qux"], - "qux": {}, - "quux": [], - } - new_dict = salt.utils.data.filter_falsey(old_dict) - expect_dict = { - "bar": {"baz": {"qux": None, "quux": "", "foo": []}}, - "baz": ["qux"], - } - self.assertEqual(expect_dict, new_dict) - self.assertIs(type(expect_dict), type(new_dict)) - - def test_filter_dict_recurse(self): - """ - Test filtering a dictionary with recursing. - This will filter out any key-values where the values are falsey or when - the values *become* falsey after filtering their contents (in case they - are lists or dicts). - """ - old_dict = { - "foo": None, - "bar": {"baz": {"qux": None, "quux": "", "foo": []}}, - "baz": ["qux"], - "qux": {}, - "quux": [], - } - new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3) - expect_dict = {"baz": ["qux"]} - self.assertEqual(expect_dict, new_dict) - self.assertIs(type(expect_dict), type(new_dict)) - - def test_filter_list_no_recurse(self): - """ - Test filtering a list without recursing. - This will only filter out items which are falsey. - """ - old_list = ["foo", None, [], {}, 0, ""] - new_list = salt.utils.data.filter_falsey(old_list) - expect_list = ["foo"] - self.assertEqual(expect_list, new_list) - self.assertIs(type(expect_list), type(new_list)) - # Ensure nested values are *not* filtered out. - old_list = [ - "foo", - ["foo"], - ["foo", None], - {"foo": 0}, - {"foo": "bar", "baz": []}, - [{"foo": ""}], - ] - new_list = salt.utils.data.filter_falsey(old_list) - self.assertEqual(old_list, new_list) - self.assertIs(type(old_list), type(new_list)) - - def test_filter_list_recurse(self): - """ - Test filtering a list with recursing. - This will filter out any items which are falsey, or which become falsey - after filtering their contents (in case they are lists or dicts). - """ - old_list = [ - "foo", - ["foo"], - ["foo", None], - {"foo": 0}, - {"foo": "bar", "baz": []}, - [{"foo": ""}], - ] - new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3) - expect_list = ["foo", ["foo"], ["foo"], {"foo": "bar"}] - self.assertEqual(expect_list, new_list) - self.assertIs(type(expect_list), type(new_list)) - - def test_filter_set_no_recurse(self): - """ - Test filtering a set without recursing. - Note that a set cannot contain unhashable types, so recursion is not possible. - """ - old_set = {"foo", None, 0, ""} - new_set = salt.utils.data.filter_falsey(old_set) - expect_set = {"foo"} - self.assertEqual(expect_set, new_set) - self.assertIs(type(expect_set), type(new_set)) - - def test_filter_ordereddict_no_recurse(self): - """ - Test filtering an OrderedDict without recursing. - """ - old_dict = OrderedDict( - [ - ("foo", None), - ( - "bar", - OrderedDict( - [ - ( - "baz", - OrderedDict([("qux", None), ("quux", ""), ("foo", [])]), - ) - ] - ), - ), - ("baz", ["qux"]), - ("qux", {}), - ("quux", []), - ] - ) - new_dict = salt.utils.data.filter_falsey(old_dict) - expect_dict = OrderedDict( - [ - ( - "bar", - OrderedDict( - [ - ( - "baz", - OrderedDict([("qux", None), ("quux", ""), ("foo", [])]), - ) - ] - ), - ), - ("baz", ["qux"]), - ] - ) - self.assertEqual(expect_dict, new_dict) - self.assertIs(type(expect_dict), type(new_dict)) - - def test_filter_ordereddict_recurse(self): - """ - Test filtering an OrderedDict with recursing. - """ - old_dict = OrderedDict( - [ - ("foo", None), - ( - "bar", - OrderedDict( - [ - ( - "baz", - OrderedDict([("qux", None), ("quux", ""), ("foo", [])]), - ) - ] - ), - ), - ("baz", ["qux"]), - ("qux", {}), - ("quux", []), - ] - ) - new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3) - expect_dict = OrderedDict([("baz", ["qux"])]) - self.assertEqual(expect_dict, new_dict) - self.assertIs(type(expect_dict), type(new_dict)) - - def test_filter_list_recurse_limit(self): - """ - Test filtering a list with recursing, but with a limited depth. - Note that the top-level is always processed, so a recursion depth of 2 - means that two *additional* levels are processed. - """ - old_list = [None, [None, [None, [None]]]] - new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=2) - self.assertEqual([[[[None]]]], new_list) - - def test_filter_dict_recurse_limit(self): - """ - Test filtering a dict with recursing, but with a limited depth. - Note that the top-level is always processed, so a recursion depth of 2 - means that two *additional* levels are processed. - """ - old_dict = { - "one": None, - "foo": {"two": None, "bar": {"three": None, "baz": {"four": None}}}, - } - new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=2) - self.assertEqual({"foo": {"bar": {"baz": {"four": None}}}}, new_dict) - - def test_filter_exclude_types(self): - """ - Test filtering a list recursively, but also ignoring (i.e. not filtering) - out certain types that can be falsey. - """ - # Ignore int, unicode - old_list = [ - "foo", - ["foo"], - ["foo", None], - {"foo": 0}, - {"foo": "bar", "baz": []}, - [{"foo": ""}], - ] - new_list = salt.utils.data.filter_falsey( - old_list, recurse_depth=3, ignore_types=[int, str] - ) - self.assertEqual( - ["foo", ["foo"], ["foo"], {"foo": 0}, {"foo": "bar"}, [{"foo": ""}]], - new_list, - ) - # Ignore list - old_list = [ - "foo", - ["foo"], - ["foo", None], - {"foo": 0}, - {"foo": "bar", "baz": []}, - [{"foo": ""}], - ] - new_list = salt.utils.data.filter_falsey( - old_list, recurse_depth=3, ignore_types=[type([])] - ) - self.assertEqual( - ["foo", ["foo"], ["foo"], {"foo": "bar", "baz": []}, []], new_list - ) - # Ignore dict - old_list = [ - "foo", - ["foo"], - ["foo", None], - {"foo": 0}, - {"foo": "bar", "baz": []}, - [{"foo": ""}], - ] - new_list = salt.utils.data.filter_falsey( - old_list, recurse_depth=3, ignore_types=[type({})] - ) - self.assertEqual(["foo", ["foo"], ["foo"], {}, {"foo": "bar"}, [{}]], new_list) - # Ignore NoneType - old_list = [ - "foo", - ["foo"], - ["foo", None], - {"foo": 0}, - {"foo": "bar", "baz": []}, - [{"foo": ""}], - ] - new_list = salt.utils.data.filter_falsey( - old_list, recurse_depth=3, ignore_types=[type(None)] - ) - self.assertEqual(["foo", ["foo"], ["foo", None], {"foo": "bar"}], new_list) - - -class FilterRecursiveDiff(TestCase): - """ - Test suite for salt.utils.data.recursive_diff - """ - - def test_list_equality(self): - """ - Test cases where equal lists are compared. - """ - test_list = [0, 1, 2] - self.assertEqual({}, salt.utils.data.recursive_diff(test_list, test_list)) - - test_list = [[0], [1], [0, 1, 2]] - self.assertEqual({}, salt.utils.data.recursive_diff(test_list, test_list)) - - def test_dict_equality(self): - """ - Test cases where equal dicts are compared. - """ - test_dict = {"foo": "bar", "bar": {"baz": {"qux": "quux"}}, "frop": 0} - self.assertEqual({}, salt.utils.data.recursive_diff(test_dict, test_dict)) - - def test_ordereddict_equality(self): - """ - Test cases where equal OrderedDicts are compared. - """ - test_dict = OrderedDict( - [ - ("foo", "bar"), - ("bar", OrderedDict([("baz", OrderedDict([("qux", "quux")]))])), - ("frop", 0), - ] - ) - self.assertEqual({}, salt.utils.data.recursive_diff(test_dict, test_dict)) - - def test_mixed_equality(self): - """ - Test cases where mixed nested lists and dicts are compared. - """ - test_data = { - "foo": "bar", - "baz": [0, 1, 2], - "bar": {"baz": [{"qux": "quux"}, {"froop", 0}]}, - } - self.assertEqual({}, salt.utils.data.recursive_diff(test_data, test_data)) - - def test_set_equality(self): - """ - Test cases where equal sets are compared. - """ - test_set = {0, 1, 2, 3, "foo"} - self.assertEqual({}, salt.utils.data.recursive_diff(test_set, test_set)) - - # This is a bit of an oddity, as python seems to sort the sets in memory - # so both sets end up with the same ordering (0..3). - set_one = {0, 1, 2, 3} - set_two = {3, 2, 1, 0} - self.assertEqual({}, salt.utils.data.recursive_diff(set_one, set_two)) - - def test_tuple_equality(self): - """ - Test cases where equal tuples are compared. - """ - test_tuple = (0, 1, 2, 3, "foo") - self.assertEqual({}, salt.utils.data.recursive_diff(test_tuple, test_tuple)) - - def test_list_inequality(self): - """ - Test cases where two inequal lists are compared. - """ - list_one = [0, 1, 2] - list_two = ["foo", "bar", "baz"] - expected_result = {"old": list_one, "new": list_two} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(list_one, list_two) - ) - expected_result = {"new": list_one, "old": list_two} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(list_two, list_one) - ) - - list_one = [0, "foo", 1, "bar"] - list_two = [1, "foo", 1, "qux"] - expected_result = {"old": [0, "bar"], "new": [1, "qux"]} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(list_one, list_two) - ) - expected_result = {"new": [0, "bar"], "old": [1, "qux"]} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(list_two, list_one) - ) - - list_one = [0, 1, [2, 3]] - list_two = [0, 1, ["foo", "bar"]] - expected_result = {"old": [[2, 3]], "new": [["foo", "bar"]]} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(list_one, list_two) - ) - expected_result = {"new": [[2, 3]], "old": [["foo", "bar"]]} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(list_two, list_one) - ) - - def test_dict_inequality(self): - """ - Test cases where two inequal dicts are compared. - """ - dict_one = {"foo": 1, "bar": 2, "baz": 3} - dict_two = {"foo": 2, 1: "bar", "baz": 3} - expected_result = {"old": {"foo": 1, "bar": 2}, "new": {"foo": 2, 1: "bar"}} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(dict_one, dict_two) - ) - expected_result = {"new": {"foo": 1, "bar": 2}, "old": {"foo": 2, 1: "bar"}} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(dict_two, dict_one) - ) - - dict_one = {"foo": {"bar": {"baz": 1}}} - dict_two = {"foo": {"qux": {"baz": 1}}} - expected_result = {"old": dict_one, "new": dict_two} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(dict_one, dict_two) - ) - expected_result = {"new": dict_one, "old": dict_two} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(dict_two, dict_one) - ) - - def test_ordereddict_inequality(self): - """ - Test cases where two inequal OrderedDicts are compared. - """ - odict_one = OrderedDict([("foo", "bar"), ("bar", "baz")]) - odict_two = OrderedDict([("bar", "baz"), ("foo", "bar")]) - expected_result = {"old": odict_one, "new": odict_two} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(odict_one, odict_two) - ) - - def test_set_inequality(self): - """ - Test cases where two inequal sets are compared. - Tricky as the sets are compared zipped, so shuffled sets of equal values - are considered different. - """ - set_one = {0, 1, 2, 4} - set_two = {0, 1, 3, 4} - expected_result = {"old": {2}, "new": {3}} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(set_one, set_two) - ) - expected_result = {"new": {2}, "old": {3}} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(set_two, set_one) - ) - - # It is unknown how different python versions will store sets in memory. - # Python 2.7 seems to sort it (i.e. set_one below becomes {0, 1, 'foo', 'bar'} - # However Python 3.6.8 stores it differently each run. - # So just test for "not equal" here. - set_one = {0, "foo", 1, "bar"} - set_two = {"foo", 1, "bar", 2} - expected_result = {} - self.assertNotEqual( - expected_result, salt.utils.data.recursive_diff(set_one, set_two) - ) - - def test_mixed_inequality(self): - """ - Test cases where two mixed dicts/iterables that are different are compared. - """ - dict_one = {"foo": [1, 2, 3]} - dict_two = {"foo": [3, 2, 1]} - expected_result = {"old": {"foo": [1, 3]}, "new": {"foo": [3, 1]}} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(dict_one, dict_two) - ) - expected_result = {"new": {"foo": [1, 3]}, "old": {"foo": [3, 1]}} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(dict_two, dict_one) - ) - - list_one = [1, 2, {"foo": ["bar", {"foo": 1, "bar": 2}]}] - list_two = [3, 4, {"foo": ["qux", {"foo": 1, "bar": 2}]}] - expected_result = { - "old": [1, 2, {"foo": ["bar"]}], - "new": [3, 4, {"foo": ["qux"]}], - } - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(list_one, list_two) - ) - expected_result = { - "new": [1, 2, {"foo": ["bar"]}], - "old": [3, 4, {"foo": ["qux"]}], - } - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(list_two, list_one) - ) - - mixed_one = {"foo": {0, 1, 2}, "bar": [0, 1, 2]} - mixed_two = {"foo": {1, 2, 3}, "bar": [1, 2, 3]} - expected_result = { - "old": {"foo": {0}, "bar": [0, 1, 2]}, - "new": {"foo": {3}, "bar": [1, 2, 3]}, - } - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(mixed_one, mixed_two) - ) - expected_result = { - "new": {"foo": {0}, "bar": [0, 1, 2]}, - "old": {"foo": {3}, "bar": [1, 2, 3]}, - } - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(mixed_two, mixed_one) - ) - - def test_tuple_inequality(self): - """ - Test cases where two tuples that are different are compared. - """ - tuple_one = (1, 2, 3) - tuple_two = (3, 2, 1) - expected_result = {"old": (1, 3), "new": (3, 1)} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(tuple_one, tuple_two) - ) - - def test_list_vs_set(self): - """ - Test case comparing a list with a set, will be compared unordered. - """ - mixed_one = [1, 2, 3] - mixed_two = {3, 2, 1} - expected_result = {} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(mixed_one, mixed_two) - ) - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(mixed_two, mixed_one) - ) - - def test_dict_vs_ordereddict(self): - """ - Test case comparing a dict with an ordereddict, will be compared unordered. - """ - test_dict = {"foo": "bar", "bar": "baz"} - test_odict = OrderedDict([("foo", "bar"), ("bar", "baz")]) - self.assertEqual({}, salt.utils.data.recursive_diff(test_dict, test_odict)) - self.assertEqual({}, salt.utils.data.recursive_diff(test_odict, test_dict)) - - test_odict2 = OrderedDict([("bar", "baz"), ("foo", "bar")]) - self.assertEqual({}, salt.utils.data.recursive_diff(test_dict, test_odict2)) - self.assertEqual({}, salt.utils.data.recursive_diff(test_odict2, test_dict)) - - def test_list_ignore_ignored(self): - """ - Test case comparing two lists with ignore-list supplied (which is not used - when comparing lists). - """ - list_one = [1, 2, 3] - list_two = [3, 2, 1] - expected_result = {"old": [1, 3], "new": [3, 1]} - self.assertEqual( - expected_result, - salt.utils.data.recursive_diff(list_one, list_two, ignore_keys=[1, 3]), - ) - - def test_dict_ignore(self): - """ - Test case comparing two dicts with ignore-list supplied. - """ - dict_one = {"foo": 1, "bar": 2, "baz": 3} - dict_two = {"foo": 3, "bar": 2, "baz": 1} - expected_result = {"old": {"baz": 3}, "new": {"baz": 1}} - self.assertEqual( - expected_result, - salt.utils.data.recursive_diff(dict_one, dict_two, ignore_keys=["foo"]), - ) - - def test_ordereddict_ignore(self): - """ - Test case comparing two OrderedDicts with ignore-list supplied. - """ - odict_one = OrderedDict([("foo", 1), ("bar", 2), ("baz", 3)]) - odict_two = OrderedDict([("baz", 1), ("bar", 2), ("foo", 3)]) - # The key 'foo' will be ignored, which means the key from the other OrderedDict - # will always be considered "different" since OrderedDicts are compared ordered. - expected_result = { - "old": OrderedDict([("baz", 3)]), - "new": OrderedDict([("baz", 1)]), - } - self.assertEqual( - expected_result, - salt.utils.data.recursive_diff(odict_one, odict_two, ignore_keys=["foo"]), - ) - - def test_dict_vs_ordereddict_ignore(self): - """ - Test case comparing a dict with an OrderedDict with ignore-list supplied. - """ - dict_one = {"foo": 1, "bar": 2, "baz": 3} - odict_two = OrderedDict([("foo", 3), ("bar", 2), ("baz", 1)]) - expected_result = {"old": {"baz": 3}, "new": OrderedDict([("baz", 1)])} - self.assertEqual( - expected_result, - salt.utils.data.recursive_diff(dict_one, odict_two, ignore_keys=["foo"]), - ) - - def test_mixed_nested_ignore(self): - """ - Test case comparing mixed, nested items with ignore-list supplied. - """ - dict_one = {"foo": [1], "bar": {"foo": 1, "bar": 2}, "baz": 3} - dict_two = {"foo": [2], "bar": {"foo": 3, "bar": 2}, "baz": 1} - expected_result = {"old": {"baz": 3}, "new": {"baz": 1}} - self.assertEqual( - expected_result, - salt.utils.data.recursive_diff(dict_one, dict_two, ignore_keys=["foo"]), - ) - - def test_ordered_dict_unequal_length(self): - """ - Test case comparing two OrderedDicts of unequal length. - """ - odict_one = OrderedDict([("foo", 1), ("bar", 2), ("baz", 3)]) - odict_two = OrderedDict([("foo", 1), ("bar", 2)]) - expected_result = {"old": OrderedDict([("baz", 3)]), "new": {}} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(odict_one, odict_two) - ) - - def test_list_unequal_length(self): - """ - Test case comparing two lists of unequal length. - """ - list_one = [1, 2, 3] - list_two = [1, 2, 3, 4] - expected_result = {"old": [], "new": [4]} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(list_one, list_two) - ) - - def test_set_unequal_length(self): - """ - Test case comparing two sets of unequal length. - This does not do anything special, as it is unordered. - """ - set_one = {1, 2, 3} - set_two = {4, 3, 2, 1} - expected_result = {"old": set(), "new": {4}} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(set_one, set_two) - ) - - def test_tuple_unequal_length(self): - """ - Test case comparing two tuples of unequal length. - This should be the same as comparing two ordered lists. - """ - tuple_one = (1, 2, 3) - tuple_two = (1, 2, 3, 4) - expected_result = {"old": (), "new": (4,)} - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(tuple_one, tuple_two) - ) - - def test_list_unordered(self): - """ - Test case comparing two lists unordered. - """ - list_one = [1, 2, 3, 4] - list_two = [4, 3, 2] - expected_result = {"old": [1], "new": []} - self.assertEqual( - expected_result, - salt.utils.data.recursive_diff(list_one, list_two, ignore_order=True), - ) - - def test_mixed_nested_unordered(self): - """ - Test case comparing nested dicts/lists unordered. - """ - dict_one = {"foo": {"bar": [1, 2, 3]}, "bar": [{"foo": 4}, 0]} - dict_two = {"foo": {"bar": [3, 2, 1]}, "bar": [0, {"foo": 4}]} - expected_result = {} - self.assertEqual( - expected_result, - salt.utils.data.recursive_diff(dict_one, dict_two, ignore_order=True), - ) - expected_result = { - "old": {"foo": {"bar": [1, 3]}, "bar": [{"foo": 4}, 0]}, - "new": {"foo": {"bar": [3, 1]}, "bar": [0, {"foo": 4}]}, - } - self.assertEqual( - expected_result, salt.utils.data.recursive_diff(dict_one, dict_two) - ) - - def test_ordered_dict_unordered(self): - """ - Test case comparing OrderedDicts unordered. - """ - odict_one = OrderedDict([("foo", 1), ("bar", 2), ("baz", 3)]) - odict_two = OrderedDict([("baz", 3), ("bar", 2), ("foo", 1)]) - expected_result = {} - self.assertEqual( - expected_result, - salt.utils.data.recursive_diff(odict_one, odict_two, ignore_order=True), - ) - - def test_ignore_missing_keys_dict(self): - """ - Test case ignoring missing keys on a comparison of dicts. - """ - dict_one = {"foo": 1, "bar": 2, "baz": 3} - dict_two = {"bar": 3} - expected_result = {"old": {"bar": 2}, "new": {"bar": 3}} - self.assertEqual( - expected_result, - salt.utils.data.recursive_diff( - dict_one, dict_two, ignore_missing_keys=True - ), - ) - - def test_ignore_missing_keys_ordered_dict(self): - """ - Test case not ignoring missing keys on a comparison of OrderedDicts. - """ - odict_one = OrderedDict([("foo", 1), ("bar", 2), ("baz", 3)]) - odict_two = OrderedDict([("bar", 3)]) - expected_result = {"old": odict_one, "new": odict_two} - self.assertEqual( - expected_result, - salt.utils.data.recursive_diff( - odict_one, odict_two, ignore_missing_keys=True - ), - ) - - def test_ignore_missing_keys_recursive(self): - """ - Test case ignoring missing keys on a comparison of nested dicts. - """ - dict_one = {"foo": {"bar": 2, "baz": 3}} - dict_two = {"foo": {"baz": 3}} - expected_result = {} - self.assertEqual( - expected_result, - salt.utils.data.recursive_diff( - dict_one, dict_two, ignore_missing_keys=True - ), - ) - # Compare from dict-in-dict - dict_two = {} - self.assertEqual( - expected_result, - salt.utils.data.recursive_diff( - dict_one, dict_two, ignore_missing_keys=True - ), - ) - # Compare from dict-in-list - dict_one = {"foo": ["bar", {"baz": 3}]} - dict_two = {"foo": ["bar", {}]} - self.assertEqual( - expected_result, - salt.utils.data.recursive_diff( - dict_one, dict_two, ignore_missing_keys=True - ), - ) From 6b334b72edfe2af92dc004734a06f2734a4ac83c Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Thu, 2 Nov 2023 12:09:45 -0600 Subject: [PATCH 25/26] Marked odict as no code-coverage, and removed some proposed tests, essentially using Python's OrderedDict --- salt/utils/odict.py | 2 ++ tests/pytests/unit/utils/test_data.py | 41 --------------------------- 2 files changed, 2 insertions(+), 41 deletions(-) diff --git a/salt/utils/odict.py b/salt/utils/odict.py index 147fa7980a8..76f313fe5be 100644 --- a/salt/utils/odict.py +++ b/salt/utils/odict.py @@ -19,6 +19,8 @@ http://stackoverflow.com/questions/6190331/ """ +# pragma: no cover # essentially using Python's OrderDict + from collections.abc import Callable diff --git a/tests/pytests/unit/utils/test_data.py b/tests/pytests/unit/utils/test_data.py index 01f9ae3cfe5..5614a770316 100644 --- a/tests/pytests/unit/utils/test_data.py +++ b/tests/pytests/unit/utils/test_data.py @@ -1456,44 +1456,3 @@ def test_ignore_missing_keys_recursive(): assert expected_result == salt.utils.data.recursive_diff( dict_one, dict_two, ignore_missing_keys=True ) - - -def test_ordereddict_dict_items(): - """ - Test case for SaltOrderedDicts items. - """ - dict_one = {"foo": 1, "bar": 2, "baz": 3} - expected_items = 'odict_items([("foo", 1), ("bar", 2), ("baz", 3)])' - - odict_one = SaltOrderedDict(dict_one) - myodict = odict_one.items() - log.warning(f"DGM test_ordered_dict_items items '{str(myodict)}'") - ## assert odict_one.items() == expected_items - - myodict = odict_one.keys() - log.warning(f"DGM test_ordered_dict_items keys '{myodict}'") - ## assert odict_one.items() == expected - - myodict = odict_one.values() - log.warning(f"DGM test_ordered_dict_items values '{myodict}'") - ## assert odict_one.items() == expected - - myodict = odict_one.popitem() - log.warning(f"DGM test_ordered_dict_items popitem '{myodict}'") - ## assert odict_one.items() == expected - - odict_one.clear() - myodict = odict_one.popitem() - log.warning(f"DGM test_ordered_dict_items clear '{myodict}'") - ## assert odict_one.items() == expected - - dgm_dir = dir(odict_one) - log.warning(f"DGM test_ordered_dict_items dgm_dir iterkeys '{dgm_dir}'") - - for myodict in odict_one.iterkeys(): - log.warning(f"DGM test_ordered_dict_items iterkeys '{myodict}'") - ## assert odict_one.items() == expected - - for myodict in odict_one.itervalues(): - log.warning(f"DGM test_ordered_dict_items itervalues '{myodict}'") - ## assert odict_one.items() == expected From 90fe04b304a9ab92bfc99e57cb77935c07477c27 Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Fri, 3 Nov 2023 09:07:14 -0600 Subject: [PATCH 26/26] Adjust break in control file to 3006.4, for updated salt-common commit b3fcdf8 --- pkg/debian/control | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/debian/control b/pkg/debian/control index 0ad39d1d9c0..c08d99d5e23 100644 --- a/pkg/debian/control +++ b/pkg/debian/control @@ -25,7 +25,7 @@ Description: Salt debug symbols Package: salt-common Architecture: amd64 arm64 Depends: ${misc:Depends} -Breaks: salt-minion (<= 3006.1) +Breaks: salt-minion (<= 3006.4) Suggests: ifupdown Recommends: lsb-release Description: shared libraries that salt requires for all packages @@ -51,8 +51,8 @@ Description: shared libraries that salt requires for all packages Package: salt-master Architecture: amd64 arm64 -Replaces: salt-common (<= 3006.1) -Breaks: salt-common (<= 3006.1) +Replaces: salt-common (<= 3006.4) +Breaks: salt-common (<= 3006.4) Depends: salt-common (= ${source:Version}), ${misc:Depends} Description: remote manager to administer servers via salt @@ -77,8 +77,8 @@ Description: remote manager to administer servers via salt Package: salt-minion Architecture: amd64 arm64 -Replaces: salt-common (<= 3006.1) -Breaks: salt-common (<= 3006.1) +Replaces: salt-common (<= 3006.4) +Breaks: salt-common (<= 3006.4) Depends: bsdmainutils, dctrl-tools, salt-common (= ${source:Version}), @@ -131,7 +131,7 @@ Description: master-of-masters for salt, the distributed remote execution system Package: salt-ssh Architecture: amd64 arm64 -Breaks: salt-common (<= 3006.3) +Breaks: salt-common (<= 3006.4) Depends: salt-common (= ${source:Version}), openssh-client, ${misc:Depends} @@ -160,7 +160,7 @@ Description: remote manager to administer servers via Salt SSH Package: salt-cloud Architecture: amd64 arm64 -Breaks: salt-common (<= 3006.3) +Breaks: salt-common (<= 3006.4) Depends: salt-common (= ${source:Version}), ${misc:Depends} Description: public cloud VM management system