From fdbb4ed333e5a9d9cbfe27efa02b68ef54f38c57 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sun, 13 Aug 2023 21:40:29 -0700 Subject: [PATCH] Update transport docs with websockt transport --- doc/topics/transports/index.rst | 2 + doc/topics/transports/ssl.rst | 73 +++++++++++++++++++ doc/topics/transports/tcp.rst | 52 +++---------- doc/topics/transports/ws.rst | 21 ++++++ salt/transport/base.py | 12 ++- salt/transport/tcp.py | 3 +- salt/transport/ws.py | 26 ++++--- tests/pytests/functional/channel/conftest.py | 8 +- .../functional/channel/test_req_channel.py | 2 +- .../functional/transport/server/conftest.py | 7 +- .../transport/server/test_publish_server.py | 7 +- .../transport/server/test_request_server.py | 1 - 12 files changed, 149 insertions(+), 65 deletions(-) create mode 100644 doc/topics/transports/ssl.rst create mode 100644 doc/topics/transports/ws.rst diff --git a/doc/topics/transports/index.rst b/doc/topics/transports/index.rst index 9b74720a96a..387852ddd00 100644 --- a/doc/topics/transports/index.rst +++ b/doc/topics/transports/index.rst @@ -38,3 +38,5 @@ The request client sends requests to a Request Server and receives a reply messa zeromq tcp + ws + ssl diff --git a/doc/topics/transports/ssl.rst b/doc/topics/transports/ssl.rst new file mode 100644 index 00000000000..86f84ba1474 --- /dev/null +++ b/doc/topics/transports/ssl.rst @@ -0,0 +1,73 @@ +Transport TLS Support +===================== + +Whenever possible transports should provide TLS Support. Currently the :doc:`tcp` and +:doc:`ws` transports support encryption and verification using TLS. + +.. versionadded:: 2016.11.1 + +The TCP transport allows for the master/minion communication to be optionally +wrapped in a TLS connection. Enabling this is simple, the master and minion need +to be using the tcp connection, then the ``ssl`` option is enabled. The ``ssl`` +option is passed as a dict and roughly corresponds to the options passed to the +Python `ssl.wrap_socket `_ +function for backwards compatability. + +.. versionadded:: 3007.0 + +The ``ssl`` option accepts ``verify_locations`` and ``verify_flags``. The +``verify_locations`` option is a list of strings or ditionaries. Strings are +passed as a single argument to the SSL context's ``load_verify_locations`` +method. Dictionaries keys are expected to be one of ``cafile``, ``capath``, +``cadata``. For each correspoding key the key and value will be passed as a +keyword argument to ``load_verify_locations``. The ``verify_flags`` options is +a list of string names of verification flags which will be set on the SSL +context. + +A simple setup looks like this, on the Salt Master add the ``ssl`` option to the +master configuration file: + +.. code-block:: yaml + + ssl: + keyfile: + certfile: + +A more complex setup looks like this, on the Salt Master add the ``ssl`` +option to the master's configuration file. In this example the Salt Master will +require valid client side certificates from Minions by setting ``cert_reqs`` to +``CERT_REQUIRED``. The Salt Master will also check a certificate revocation list +if one is provided in ``verify_locations``: + +.. code-block:: yaml + + ssl: + keyfile: + certfile: + cert_reqs: CERT_REQUIRED + verify_locations: + - + - capath: + - cafile: + verify_flags: + - VERIFY_CRL_CHECK_CHAIN + + +The minimal `ssl` option in the minion configuration file looks like this: + +.. code-block:: yaml + + ssl: True + # Versions below 2016.11.4: + ssl: {} + +A Minion can be configured to present a client certificat to the master like this: + +.. code-block:: yaml + + ssl: + keyfile: + certfile: + +Specific options can be sent to the minion also, as defined in the Python +`ssl.wrap_socket` function. diff --git a/doc/topics/transports/tcp.rst b/doc/topics/transports/tcp.rst index 267b6fb012c..a0443e345f7 100644 --- a/doc/topics/transports/tcp.rst +++ b/doc/topics/transports/tcp.rst @@ -19,6 +19,14 @@ to ``tcp`` on each Salt minion and Salt master. use the same transport. We're investigating a report of an error when using mixed transport types at very heavy loads. + +TLS Support +=========== + +The TLS transport support full encryption and verification using both server +and client certificates. See :doc:`ssl` for more details. + + Wire Protocol ============= This implementation over TCP focuses on flexibility over absolute efficiency. @@ -37,51 +45,9 @@ actual message that we are sending. With this flexible wire protocol we can implement any message semantics that we'd like-- including multiplexed message passing on a single socket. -TLS Support -=========== - -.. versionadded:: 2016.11.1 - -The TCP transport allows for the master/minion communication to be optionally -wrapped in a TLS connection. Enabling this is simple, the master and minion need -to be using the tcp connection, then the `ssl` option is enabled. The `ssl` -option is passed as a dict and corresponds to the options passed to the -Python `ssl.wrap_socket `_ -function. - -A simple setup looks like this, on the Salt Master add the `ssl` option to the -master configuration file: - -.. code-block:: yaml - - ssl: - keyfile: - certfile: - ssl_version: PROTOCOL_TLSv1_2 - ciphers: ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384 - -The minimal `ssl` option in the minion configuration file looks like this: - -.. code-block:: yaml - - ssl: True - # Versions below 2016.11.4: - ssl: {} - -Specific options can be sent to the minion also, as defined in the Python -`ssl.wrap_socket` function. - -.. note:: - - While setting the ssl_version is not required, we recommend it. Some older - versions of python do not support the latest TLS protocol and if this is - the case for your version of python we strongly recommend upgrading your - version of Python. Ciphers specification might be omitted, but strongly - recommended as otherwise all available ciphers will be enabled. - - Crypto ====== + The current implementation uses the same crypto as the ``zeromq`` transport. diff --git a/doc/topics/transports/ws.rst b/doc/topics/transports/ws.rst new file mode 100644 index 00000000000..742e40c5f32 --- /dev/null +++ b/doc/topics/transports/ws.rst @@ -0,0 +1,21 @@ +=================== +Websocket Transport +=================== + +The Websocket transport is an implementation of Salt's transport using the websocket protocol. +The Websocket transport is enabled by changing the :conf_minion:`transport` setting +to ``ws`` on each Salt minion and Salt master. + +TLS Support +=========== + +The Websocket transport support full encryption and verification using both server +and client certificates. See :doc:`ssl` for more details. + +Publish Server and Client +========================= +The publish server and client are implemented using aiohttp. + +Request Server and Client +========================= +The request server and client are implemented using aiohttp. diff --git a/salt/transport/base.py b/salt/transport/base.py index 364b3b5badc..178d4ac13d8 100644 --- a/salt/transport/base.py +++ b/salt/transport/base.py @@ -436,12 +436,22 @@ class PublishClient(Transport): def ssl_context(ssl_options, server_side=False): + """ + Create an ssl context from the provided ssl_options. This method preserves + backwards compatability older ssl config settings but adds verify_locations + and verify_flags options. + """ default_version = ssl.PROTOCOL_TLS if server_side: default_version = ssl.PROTOCOL_TLS_SERVER + purpose = ssl.Purpose.CLIENT_AUTH elif server_side is not None: default_version = ssl.PROTOCOL_TLS_CLIENT - context = ssl.SSLContext(ssl_options.get("ssl_version", default_version)) + purpose = ssl.Purpose.SERVER_AUTH + # Use create_default_context to start with what Python considers resonably + # secure settings. + context = ssl.create_default_context(purpose) + context.protocol = ssl_options.get("ssl_version", default_version) if "certfile" in ssl_options: context.load_cert_chain( ssl_options["certfile"], ssl_options.get("keyfile", None) diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py index f10fbf09e48..8998ab4591f 100644 --- a/salt/transport/tcp.py +++ b/salt/transport/tcp.py @@ -1454,7 +1454,8 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer): process_manager.add_process( self.publish_daemon, args=[self.publish_payload], - name=self.__class__.__name__) + name=self.__class__.__name__, + ) async def publish_payload(self, payload, *args): return await self.pub_server.publish_payload(payload) diff --git a/salt/transport/ws.py b/salt/transport/ws.py index ce377ac8925..3522c5b4cb9 100644 --- a/salt/transport/ws.py +++ b/salt/transport/ws.py @@ -4,7 +4,6 @@ import multiprocessing import socket import time import warnings -import functools import aiohttp import aiohttp.web @@ -365,17 +364,21 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer): await runner.setup() site = aiohttp.web.SockSite(runner, sock, ssl_context=ctx) log.info("Publisher binding to socket %s:%s", self.pub_host, self.pub_port) - print('start site') + print("start site") await site.start() - print('start puller') + print("start puller") self._pub_payload = publish_payload if self.pull_path: with salt.utils.files.set_umask(0o177): - self.puller = await asyncio.start_unix_server(self.pull_handler, self.pull_path) + self.puller = await asyncio.start_unix_server( + self.pull_handler, self.pull_path + ) else: - self.puller = await asyncio.start_server(self.pull_handler, self.pull_host, self.pull_port) - print('puller started') + self.puller = await asyncio.start_server( + self.pull_handler, self.pull_host, self.pull_port + ) + print("puller started") while self._run.is_set(): await asyncio.sleep(0.3) await self.server.stop() @@ -399,7 +402,8 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer): process_manager.add_process( self.publish_daemon, args=[self.publish_payload], - name=self.__class__.__name__) + name=self.__class__.__name__, + ) async def handle_request(self, request): try: @@ -418,9 +422,13 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer): async def _connect(self): if self.pull_path: - self.pub_reader, self.pub_writer = await asyncio.open_unix_connection(self.pull_path) + self.pub_reader, self.pub_writer = await asyncio.open_unix_connection( + self.pull_path + ) else: - self.pub_reader, self.pub_writer = await asyncio.open_connection(self.pull_host, self.pull_port) + self.pub_reader, self.pub_writer = await asyncio.open_connection( + self.pull_host, self.pull_port + ) self._connecting = None def connect(self): diff --git a/tests/pytests/functional/channel/conftest.py b/tests/pytests/functional/channel/conftest.py index 4aeb7956ee9..387e3bcf4e5 100644 --- a/tests/pytests/functional/channel/conftest.py +++ b/tests/pytests/functional/channel/conftest.py @@ -28,7 +28,7 @@ def _prepare_aes(): def transport_ids(value): - return "Transport({})".format(value) + return f"Transport({value})" @pytest.fixture(params=("zeromq", "tcp"), ids=transport_ids) @@ -44,7 +44,7 @@ def salt_master(salt_factories, transport): "sign_pub_messages": False, } factory = salt_factories.salt_master_daemon( - random_string("server-{}-master-".format(transport)), + random_string(f"server-{transport}-master-"), defaults=config_defaults, ) return factory @@ -58,10 +58,10 @@ def salt_minion(salt_master, transport): "master_port": salt_master.config["ret_port"], "auth_timeout": 5, "auth_tries": 1, - "master_uri": "tcp://127.0.0.1:{}".format(salt_master.config["ret_port"]), + "master_uri": f"tcp://127.0.0.1:{salt_master.config['ret_port']}", } factory = salt_master.salt_minion_daemon( - random_string("server-{}-minion-".format(transport)), + random_string("server-{transport}-minion-"), defaults=config_defaults, ) return factory diff --git a/tests/pytests/functional/channel/test_req_channel.py b/tests/pytests/functional/channel/test_req_channel.py index 1ed69355baf..32aa4e11fbb 100644 --- a/tests/pytests/functional/channel/test_req_channel.py +++ b/tests/pytests/functional/channel/test_req_channel.py @@ -113,7 +113,7 @@ def req_server_channel(salt_master, req_channel_crypt): def req_channel_crypt_ids(value): - return "ReqChannel(crypt='{}')".format(value) + return f"ReqChannel(crypt='{value}')" @pytest.fixture(params=["clear", "aes"], ids=req_channel_crypt_ids) diff --git a/tests/pytests/functional/transport/server/conftest.py b/tests/pytests/functional/transport/server/conftest.py index 7c8d650d1bb..80ad4c55531 100644 --- a/tests/pytests/functional/transport/server/conftest.py +++ b/tests/pytests/functional/transport/server/conftest.py @@ -1,9 +1,10 @@ -import salt.utils.process - import pytest +import salt.utils.process + + def transport_ids(value): - return "Transport({})".format(value) + return f"Transport({value})" @pytest.fixture(params=("zeromq", "tcp", "ws"), ids=transport_ids) diff --git a/tests/pytests/functional/transport/server/test_publish_server.py b/tests/pytests/functional/transport/server/test_publish_server.py index 2688ae512af..0df01888bd0 100644 --- a/tests/pytests/functional/transport/server/test_publish_server.py +++ b/tests/pytests/functional/transport/server/test_publish_server.py @@ -1,4 +1,5 @@ import asyncio + import salt.transport @@ -11,7 +12,9 @@ async def test_publsh_server( pub_server.pre_fork(process_manager) await asyncio.sleep(3) - pub_client = salt.transport.publish_client(minion_opts, io_loop, master_opts["interface"], master_opts["publish_port"]) + pub_client = salt.transport.publish_client( + minion_opts, io_loop, master_opts["interface"], master_opts["publish_port"] + ) await pub_client.connect() # Yield to loop in order to allow pub client to connect. @@ -34,4 +37,4 @@ async def test_publsh_server( pub_client.close() # Yield to loop in order to allow background close methods to finish. - await asyncio.sleep(.3) + await asyncio.sleep(0.3) diff --git a/tests/pytests/functional/transport/server/test_request_server.py b/tests/pytests/functional/transport/server/test_request_server.py index 88bc9966f75..773de615e4d 100644 --- a/tests/pytests/functional/transport/server/test_request_server.py +++ b/tests/pytests/functional/transport/server/test_request_server.py @@ -1,6 +1,5 @@ import asyncio - import salt.transport import salt.utils.process