diff --git a/HACKING.rst b/HACKING.rst index 4c17e671b9a..05a8b41ca09 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -155,15 +155,6 @@ ZeroMQ Transport: pip install -e . -RAET Transport: - -.. code-block:: bash - - pip install -r requirements/raet.txt - pip install psutil - pip install -e . - - Running a self-contained development version ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/.tx/config b/doc/.tx/config index 1caedaff7a9..f766c4a9100 100644 --- a/doc/.tx/config +++ b/doc/.tx/config @@ -4112,12 +4112,6 @@ source_file = _build/locale/topics/development/hacking.pot source_lang = en source_name = topics/development/hacking.rst -[salt.topics--development--raet--index] -file_filter = locale//LC_MESSAGES/topics/development/raet/index.po -source_file = _build/locale/topics/development/raet/index.pot -source_lang = en -source_name = topics/development/raet/index.rst - [salt.topics--development--salt_projects] file_filter = locale//LC_MESSAGES/topics/development/salt_projects.po source_file = _build/locale/topics/development/salt_projects.pot @@ -4184,18 +4178,6 @@ source_file = _build/locale/topics/topology/syndic.pot source_lang = en source_name = topics/topology/syndic.rst -[salt.topics--transports--raet--index] -file_filter = locale//LC_MESSAGES/topics/transports/raet/index.po -source_file = _build/locale/topics/transports/raet/index.pot -source_lang = en -source_name = topics/transports/raet/index.rst - -[salt.topics--transports--raet--programming_intro] -file_filter = locale//LC_MESSAGES/topics/transports/raet/programming_intro.po -source_file = _build/locale/topics/transports/raet/programming_intro.pot -source_lang = en -source_name = topics/transports/raet/programming_intro.rst - [salt.topics--tutorials--states_pt5] file_filter = locale//LC_MESSAGES/topics/tutorials/states_pt5.po source_file = _build/locale/topics/tutorials/states_pt5.pot @@ -4652,12 +4634,6 @@ source_file = _build/locale/ref/modules/all/salt.modules.pyenv.pot source_lang = en source_name = ref/modules/all/salt.modules.pyenv.rst -[salt.ref--modules--all--salt_modules_raet_publish] -file_filter = locale//LC_MESSAGES/ref/modules/all/salt.modules.raet_publish.po -source_file = _build/locale/ref/modules/all/salt.modules.raet_publish.pot -source_lang = en -source_name = ref/modules/all/salt.modules.raet_publish.rst - [salt.ref--modules--all--salt_modules_schedule] file_filter = locale//LC_MESSAGES/ref/modules/all/salt.modules.schedule.po source_file = _build/locale/ref/modules/all/salt.modules.schedule.pot diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 726a9fb2732..d0d2cf6f9d2 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -866,9 +866,8 @@ Default: ``zeromq`` Changes the underlying transport layer. ZeroMQ is the recommended transport while additional transport layers are under development. Supported values are -``zeromq``, ``raet`` (experimental), and ``tcp`` (experimental). This setting has -a significant impact on performance and should not be changed unless you know -what you are doing! +``zeromq`` and ``tcp`` (experimental). This setting has a significant impact on +performance and should not be changed unless you know what you are doing! .. code-block:: yaml diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index 30343ebd8ec..6f5acaffde4 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -1367,9 +1367,8 @@ Default: ``zeromq`` Changes the underlying transport layer. ZeroMQ is the recommended transport while additional transport layers are under development. Supported values are -``zeromq``, ``raet`` (experimental), and ``tcp`` (experimental). This setting has -a significant impact on performance and should not be changed unless you know -what you are doing! +``zeromq`` and ``tcp`` (experimental). This setting has a significant impact +on performance and should not be changed unless you know what you are doing! .. code-block:: yaml diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index f1a85d2c715..94a42d8dad8 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -362,7 +362,6 @@ execution modules qemu_nbd quota rabbitmq - raet_publish rallydev random_org rbac_solaris diff --git a/doc/ref/modules/all/salt.modules.raet_publish.rst b/doc/ref/modules/all/salt.modules.raet_publish.rst deleted file mode 100644 index fcc34841d99..00000000000 --- a/doc/ref/modules/all/salt.modules.raet_publish.rst +++ /dev/null @@ -1,6 +0,0 @@ -========================= -salt.modules.raet_publish -========================= - -.. automodule:: salt.modules.raet_publish - :members: \ No newline at end of file diff --git a/doc/topics/development/index.rst b/doc/topics/development/index.rst index 65a7e6ee861..cabc1a2ef39 100644 --- a/doc/topics/development/index.rst +++ b/doc/topics/development/index.rst @@ -10,7 +10,6 @@ Developing Salt modules/index extend/index tests/* - raet/index git/index conventions/index ../../ref/internals/index diff --git a/doc/topics/development/raet/index.rst b/doc/topics/development/raet/index.rst deleted file mode 100644 index d69cee457e3..00000000000 --- a/doc/topics/development/raet/index.rst +++ /dev/null @@ -1,272 +0,0 @@ -raet -==== - -# RAET -# Reliable Asynchronous Event Transport Protocol - -.. seealso:: :ref:`RAET Overview ` - -Protocol --------- - -Layering: - -OSI Layers - -7: Application: Format: Data (Stack to Application interface buffering etc) -6: Presentation: Format: Data (Encrypt-Decrypt convert to machine independent format) -5: Session: Format: Data (Interhost communications. Authentication. Groups) -4: Transport: Format: Segments (Reliable delivery of Message, Transactions, Segmentation, Error checking) -3: Network: Format: Packets/Datagrams (Addressing Routing) -2: Link: Format: Frames ( Reliable per frame communications connection, Media access controller ) -1: Physical: Bits (Transceiver communication connection not reliable) - -Link is hidden from Raet -Network is IP host address and Udp Port -Transport is Raet transactions, service kind, tail error checking, -Could include header signing as part of transport reliable delivery serialization of header -Session is session id key exchange for signing. Grouping is Road (like 852 channel) -Presentation is Encrypt Decrypt body Serialize Deserialize Body -Application is body data dictionary - -Header signing spans both the Transport and Session layers. - -Header ------- - -JSON Header (Tradeoff some processing speed for extensibility, ease of use, readability) - -Body initially JSON but support for "packed" binary body - - -Packet ------- - -Header ASCII Safe JSON -Header termination: -Empty line given by double pair of carriage return linefeed -/r/n/r/n -10 13 10 13 -ADAD -1010 1101 1010 1101 - -In json carriage return and newline characters cannot appear in a json encoded -string unless they are escaped with backslash, so the 4 byte combination is illegal in valid -json that does not have multi-byte unicode characters. - -These means the header must be ascii safe so no multibyte utf-8 strings -allowed in header. - -Following Header Terminator is variable length signature block. This is binary -and the length is provided in the header. - -Following the signature block is the packet body or data. -This may either be JSON or packed binary. -The format is given in the json header - -Finally is an optional tail block for error checking or encryption details - - -Header Fields -------------- - -In UDP header - -sh = source host -sp = source port -dh = destination host -dp = destination port - - -In RAET Header - -hk = header kind -hl = header length - -vn = version number - -sd = Source Device ID -dd = Destination Device ID -cf = Corresponder Flag -mf = Multicast Flag - -si = Session ID -ti = Transaction ID - -sk = Service Kind -pk = Packet Kind -bf = Burst Flag (Send all Segments or Ordered packets without interleaved acks) - -oi = Order Index -dt = DateTime Stamp - -sn = Segment Number -sc = Segment Count - -pf = Pending Segment Flag -af = All Flag (Resent all Segments not just one) - -nk = Auth header kind -nl = Auth header length - -bk = body kind -bl = body length - -tk = tail kind -tl = tail length - -fg = flags packed (Flags) Default '00' hex string - 2 byte Hex string with bits (0, 0, af, pf, 0, bf, mf, cf) - Zeros are TBD flags - - -Session Bootstrap ------------------ - -Minion sends packet with SID of Zero with public key of minions Public Private Key pair -Master acks packet with SID of Zero to let minion know it received the request - -Some time later Master sends packet with SID of zero that accepts the Minion - -Minion - - -Session -------- -Session is important for security. Want one session opened and then multiple -transactions within session. - -Session ID -SID -sid - -GUID hash to guarantee uniqueness since no guarantee of nonvolatile storage -or require file storage to keep last session ID used. - -Service Types or Modular Services ---------------------------------- -Four Service Types - -A) One or more maybe (unacknowledged repeat) maybe means no guarantee - -B) Exactly one at most (ack with retries) (duplicate detection idempotent) - at most means fixed number of retries has finite probability of failing - B1) finite retries - B2) infinite retries with exponential back-off up to a maximum delay - -C) Exactly one of sequence at most (sequence numbered) - Receiver requests retry of missing packet with same B1 or B2 retry type - -D) End to End (Application layer Request Response) - This is two B sub transactions - -Initially unicast messaging -Eventually support for Multicast - -The use case for C) is to fragment large packets as once a UDP packet -exceeds the frame size its reliability goes way down -So its more reliable to fragment large packets. - - -Better approach might be to have more modularity. -Services Levels - - 1) Maybe one or more - A) Fire and forget - no transaction either side - B) Repeat, no ack, no dupdet - repeat counter send side, - no transaction on receive side - C) Repeat, no Ack, dupdet - repeat counter send side, - dup detection transaction receive side - 2) More or Less Once - A) retry finite, ack no dupdet - retry timer send side, finite number of retires - ack receive side no dupdet - 3) At most Once - A) retry finite, ack, dupdet - retry timer send side, finite number of retires - ack receive side dupdet - 4) Exactly once - A) ack retry - retry timer send side, - ack and duplicate detection receive side - Infinite retries with exponential backoff - 5) Sequential sequence number - A) reorder escrow - B) Segmented packets - 6) request response to application layer - - - -Service Features - -1) repeats -2) ack retry transaction id -3) sequence number duplicate detection out of order detection sequencing -4) rep-req - - -Always include transaction id since multiple transactions on same port -So get duplicate detection for free if keep transaction alive but if use - - -A) Maybe one or more -B1) At Least One -B2) Exactly One -C) One of sequence -D) End to End - -A) Sender creates transaction id for number of repeats but receiver does not -keep transaction alive - -B1) Sender creates transaction id keeps it for retries. -Receiver keeps it to send ack then kills so retry could be duplicate not detected - -B2) Sender creates transaction id keeps for retries -Receiver keeps tid for acks on any retires so no duplicates. - -C) Sender creates TID and Sequence Number. -Receiver checks for out of order sequence and can request retry. - -D) Application layer sends response. So question is do we keep transaction open -or have response be new transaction. No because then we need a rep-req ID so -might as well use the same transaction id. Just keep alive until get response. - -Little advantage to B1 vs B2 not having duplicates. - -So 4 service types - -A) Maybe one or more (unacknowledged repeat) - -B) Exactly One (At most one) (ack with retry) (duplicate detection idempotent) - -C) One of Sequence (sequence numbered) - -D) End to End - - -Also multicast or unicast - - -Modular Transaction Table - -Sender Side: - Transaction ID plus transaction source sender or receiver generated transaction id - Repeat Counter - Retry Timer Retry Counter (finite retries) - Redo Timer (infinite redos with exponential backoff) - Sequence number without acks (look for resend requests) - Sequence with ack (wait for ack before sending next in sequence) - Segmentation - -Receiver Side: - Nothing just accept packet - Acknowledge (can delete transaction after acknowledge) - No duplicate detection - Transaction timeout (keep transaction until timeout) - Duplicate detection save transaction id duplicate detection timeout - Request resend of missing packet in sequence - Sequence reordering with escrow timeout wait escrow before requesting resend - Unsegmentation (request resends of missing segment) diff --git a/doc/topics/installation/index.rst b/doc/topics/installation/index.rst index 3261c184f79..3a79e2898e1 100644 --- a/doc/topics/installation/index.rst +++ b/doc/topics/installation/index.rst @@ -97,27 +97,20 @@ Salt should run on any Unix-like platform so long as the dependencies are met. * `Tornado`_ - Web framework and asynchronous networking library * `futures`_ - Python2 only dependency. Backport of the concurrent.futures package from Python 3.2 -Depending on the chosen Salt transport, `ZeroMQ`_ or `RAET`_, dependencies -vary: - * ZeroMQ: * `ZeroMQ`_ >= 3.2.0 * `pyzmq`_ >= 2.2.0 - ZeroMQ Python bindings * `PyCrypto`_ - The Python cryptography toolkit -* RAET: - * `libnacl`_ - Python bindings to `libsodium`_ - * `ioflo`_ - The flo programming interface raet and salt-raet is built on - * `RAET`_ - The worlds most awesome UDP protocol - -Salt defaults to the `ZeroMQ`_ transport, and the choice can be made at install -time, for example: +Salt defaults to the `ZeroMQ`_ transport. The ``--salt-transport`` installation +option is available, but currently only supports the ``szeromq`` option. This +may be expanded in the future. .. code-block:: bash - python setup.py --salt-transport=raet install + python setup.py --salt-transport=zeromq install This way, only the required dependencies are pulled by the setup script if need be. @@ -127,7 +120,7 @@ provided like: .. code-block:: bash - pip install --install-option="--salt-transport=raet" salt + pip install --install-option="--salt-transport=zeromq" salt .. note:: Salt does not bundle dependencies that are typically distributed as part of @@ -156,10 +149,6 @@ Optional Dependencies .. _`apache-libcloud`: http://libcloud.apache.org .. _`Requests`: http://docs.python-requests.org/en/latest .. _`Tornado`: http://www.tornadoweb.org/en/stable/ -.. _`libnacl`: https://github.com/saltstack/libnacl -.. _`ioflo`: https://github.com/ioflo/ioflo -.. _`RAET`: https://github.com/saltstack/raet -.. _`libsodium`: https://github.com/jedisct1/libsodium .. _`futures`: https://github.com/agronholm/pythonfutures diff --git a/doc/topics/transports/index.rst b/doc/topics/transports/index.rst index 7702a7eb8e3..00ed0099feb 100644 --- a/doc/topics/transports/index.rst +++ b/doc/topics/transports/index.rst @@ -34,4 +34,3 @@ guarantee minion-master confidentiality. zeromq tcp - raet/index diff --git a/doc/topics/transports/raet/index.rst b/doc/topics/transports/raet/index.rst deleted file mode 100644 index 1e0ddad3247..00000000000 --- a/doc/topics/transports/raet/index.rst +++ /dev/null @@ -1,145 +0,0 @@ -.. _raet: - -================== -The RAET Transport -================== - -.. note:: - - The RAET transport is in very early development, it is functional but no - promises are yet made as to its reliability or security. - As for reliability and security, the encryption used has been audited and - our tests show that raet is reliable. With this said we are still conducting - more security audits and pushing the reliability. - This document outlines the encryption used in RAET - -.. versionadded:: 2014.7.0 - -The Reliable Asynchronous Event Transport, or RAET, is an alternative transport -medium developed specifically with Salt in mind. It has been developed to -allow queuing to happen up on the application layer and comes with socket -layer encryption. It also abstracts a great deal of control over the socket -layer and makes it easy to bubble up errors and exceptions. - -RAET also offers very powerful message routing capabilities, allowing for -messages to be routed between processes on a single machine all the way up to -processes on multiple machines. Messages can also be restricted, allowing -processes to be sent messages of specific types from specific sources -allowing for trust to be established. - -Using RAET in Salt -================== - -Using RAET in Salt is easy, the main difference is that the core dependencies -change, instead of needing pycrypto, M2Crypto, ZeroMQ, and PYZMQ, the packages -`libsodium`_, libnacl, ioflo, and raet are required. Encryption is handled very cleanly -by libnacl, while the queueing and flow control is handled by -ioflo. Distribution packages are forthcoming, but `libsodium`_ can be easily -installed from source, or many distributions do ship packages for it. -The libnacl and ioflo packages can be easily installed from pypi, distribution -packages are in the works. - -Once the new deps are installed the 2014.7 release or higher of Salt needs to -be installed. - -Once installed, modify the configuration files for the minion and master to -set the transport to raet: - -``/etc/salt/master``: - -.. code-block:: yaml - - transport: raet - - -``/etc/salt/minion``: - -.. code-block:: yaml - - transport: raet - - -Now start salt as it would normally be started, the minion will connect to the -master and share long term keys, which can then in turn be managed via -salt-key. Remote execution and salt states will function in the same way as -with Salt over ZeroMQ. - -Limitations -=========== - -The 2014.7 release of RAET is not complete! The Syndic and Multi Master have -not been completed yet and these are slated for completion in the 2015.5.0 -release. - -Also, Salt-Raet allows for more control over the client but these hooks have -not been implemented yet, thereforre the client still uses the same system -as the ZeroMQ client. This means that the extra reliability that RAET exposes -has not yet been implemented in the CLI client. - -Why? -==== - -Customer and User Request -------------------------- - -Why make an alternative transport for Salt? There are many reasons, but the -primary motivation came from customer requests, many large companies came with -requests to run Salt over an alternative transport, the reasoning was varied, -from performance and scaling improvements to licensing concerns. These -customers have partnered with SaltStack to make RAET a reality. - -More Capabilities ------------------ - -RAET has been designed to allow salt to have greater communication -capabilities. It has been designed to allow for development into features -which out ZeroMQ topologies can't match. - -Many of the proposed features are still under development and will be -announced as they enter proof of concept phases, but these features include -`salt-fuse` - a filesystem over salt, `salt-vt` - a parallel api driven shell -over the salt transport and many others. - -RAET Reliability -================ - -RAET is reliable, hence the name (Reliable Asynchronous Event Transport). - -The concern posed by some over RAET reliability is based on the fact that -RAET uses UDP instead of TCP and UDP does not have built in reliability. - -RAET itself implements the needed reliability layers that are not natively -present in UDP, this allows RAET to dynamically optimize packet delivery -in a way that keeps it both reliable and asynchronous. - -RAET and ZeroMQ -=============== - -When using RAET, ZeroMQ is not required. RAET is a complete networking -replacement. It is noteworthy that RAET is not a ZeroMQ replacement in a -general sense, the ZeroMQ constructs are not reproduced in RAET, but they are -instead implemented in such a way that is specific to Salt's needs. - -RAET is primarily an async communication layer over truly async connections, -defaulting to UDP. ZeroMQ is over TCP and abstracts async constructs within the -socket layer. - -Salt is not dropping ZeroMQ support and has no immediate plans to do so. - -Encryption -========== - -RAET uses Dan Bernstein's NACL encryption libraries and `CurveCP`_ handshake. -The libnacl python binding binds to both `libsodium`_ and tweetnacl to execute -the underlying cryptography. This allows us to completely rely on an -externally developed cryptography system. - -Programming Intro -================= - -.. toctree:: - - programming_intro - -.. _libsodium: http://doc.libsodium.org/ -.. _CurveCP: http://curvecp.org/ diff --git a/doc/topics/transports/raet/programming_intro.rst b/doc/topics/transports/raet/programming_intro.rst deleted file mode 100644 index 770d63b63aa..00000000000 --- a/doc/topics/transports/raet/programming_intro.rst +++ /dev/null @@ -1,41 +0,0 @@ -.. _raet-programming: - -========================= -Intro to RAET Programming -========================= - -.. note:: - - This page is still under construction - -The first thing to cover is that RAET does not present a socket api, it -presents, and queueing api, all messages in RAET are made available to via -queues. This is the single most differentiating factor with RAET vs other -networking libraries, instead of making a socket, a stack is created. -Instead of calling send() or recv(), messages are placed on the stack to be -sent and messages that are received appear on the stack. - -Different kinds of stacks are also available, currently two stacks exist, -the UDP stack, and the UXD stack. The UDP stack is used to communicate over -udp sockets, and the UXD stack is used to communicate over Unix Domain -Sockets. - -The UDP stack runs a context for communicating over networks, while the -UXD stack has contexts for communicating between processes. - -UDP Stack Messages -================== - -To create a UDP stack in RAET, simply create the stack, manage the queues, -and process messages: - -.. code-block:: python - - from salt.transport.road.raet import stacking - from salt.transport.road.raet import estating - - udp_stack = stacking.StackUdp(ha=('127.0.0.1', 7870)) - r_estate = estating.Estate(stack=stack, name='foo', ha=('192.168.42.42', 7870)) - msg = {'hello': 'world'} - udp_stack.transmit(msg, udp_stack.estates[r_estate.name]) - udp_stack.serviceAll() diff --git a/salt/auth/__init__.py b/salt/auth/__init__.py index cf5e2deb9c3..d9b1a64c0c1 100644 --- a/salt/auth/__init__.py +++ b/salt/auth/__init__.py @@ -687,18 +687,12 @@ class Resolver(object): self.auth = salt.loader.auth(opts) def _send_token_request(self, load): - if self.opts['transport'] in ('zeromq', 'tcp'): - master_uri = 'tcp://' + salt.utils.zeromq.ip_bracket(self.opts['interface']) + \ - ':' + six.text_type(self.opts['ret_port']) - channel = salt.transport.client.ReqChannel.factory(self.opts, - crypt='clear', - master_uri=master_uri) - return channel.send(load) - - elif self.opts['transport'] == 'raet': - channel = salt.transport.client.ReqChannel.factory(self.opts) - channel.dst = (None, None, 'local_cmd') - return channel.send(load) + master_uri = 'tcp://' + salt.utils.zeromq.ip_bracket(self.opts['interface']) + \ + ':' + six.text_type(self.opts['ret_port']) + channel = salt.transport.client.ReqChannel.factory(self.opts, + crypt='clear', + master_uri=master_uri) + return channel.send(load) def cli(self, eauth): ''' diff --git a/salt/cli/caller.py b/salt/cli/caller.py index bcdc306ef2d..a7bda65c350 100644 --- a/salt/cli/caller.py +++ b/salt/cli/caller.py @@ -9,7 +9,6 @@ from __future__ import absolute_import, print_function, unicode_literals import os import sys -import time import logging import traceback @@ -24,27 +23,11 @@ import salt.transport.client import salt.utils.args import salt.utils.files import salt.utils.jid -import salt.utils.kinds as kinds import salt.utils.minion import salt.utils.profile import salt.utils.stringutils import salt.defaults.exitcodes -from salt.cli import daemons from salt.log import LOG_LEVELS -from salt.utils.platform import is_windows -from salt.utils.process import MultiprocessingProcess - -try: - from raet import raeting, nacling - from raet.lane.stacking import LaneStack - from raet.lane.yarding import RemoteYard, Yard - - if is_windows(): - import win32file - -except ImportError: - # Don't die on missing transport libs since only one transport is required - pass # Import 3rd-party libs from salt.ext import six @@ -78,10 +61,8 @@ class Caller(object): # switch on available ttypes if ttype in ('zeromq', 'tcp', 'detect'): return ZeroMQCaller(opts, **kwargs) - elif ttype == 'raet': - return RAETCaller(opts, **kwargs) else: - raise Exception('Callers are only defined for ZeroMQ and raet') + raise Exception('Callers are only defined for ZeroMQ and TCP') # return NewKindOfCaller(opts, **kwargs) @@ -317,7 +298,6 @@ class BaseCaller(object): # Local job cache has been enabled salt.utils.minion.cache_jobs(self.opts, ret['jid'], ret) - # close raet channel here return ret @@ -343,183 +323,3 @@ class ZeroMQCaller(BaseCaller): channel.send(load) finally: channel.close() - - -def raet_minion_run(cleanup_protecteds): - ''' - Set up the minion caller. Should be run in its own process. - This function is intentionally left out of RAETCaller. This will avoid - needing to pickle the RAETCaller object on Windows. - ''' - minion = daemons.Minion() # daemonizes here - minion.call(cleanup_protecteds=cleanup_protecteds) # caller minion.call_in uses caller.flo - - -class RAETCaller(BaseCaller): - ''' - Object to wrap the calling of local salt modules for the salt-call command - when transport is raet - - There are two operation modes. - 1) Use a preexisting minion - 2) Set up a special caller minion if no preexisting minion - The special caller minion is a subset whose only function is to perform - Salt-calls with raet as the transport - The essentials: - A RoadStack whose local estate name is of the form "role_kind" where: - role is the minion id opts['id'] - kind is opts['__role'] which should be 'caller' APPL_KIND_NAMES - The RoadStack if for communication to/from a master - - A LaneStack with manor yard so that RaetChannels created by the func Jobbers - can communicate through this manor yard then through the - RoadStack to/from a master - - A Router to route between the stacks (Road and Lane) - - These are all managed via a FloScript named caller.flo - - ''' - def __init__(self, opts): - ''' - Pass in the command line options - ''' - self.process = None - if not opts['local']: - self.stack = self._setup_caller_stack(opts) - salt.transport.jobber_stack = self.stack - - if (opts.get('__role') == - kinds.APPL_KIND_NAMES[kinds.applKinds.caller]): - # spin up and fork minion here - self.process = MultiprocessingProcess(target=raet_minion_run, - kwargs={'cleanup_protecteds': [self.stack.ha], }) - self.process.start() - # wait here until '/var/run/salt/minion/alpha_caller.manor.uxd' exists - self._wait_caller(opts) - - super(RAETCaller, self).__init__(opts) - - def run(self): - ''' - Execute the salt call logic - ''' - try: - ret = self.call() - if not self.opts['local']: - self.stack.server.close() - salt.transport.jobber_stack = None - - if self.opts['print_metadata']: - print_ret = ret - else: - print_ret = ret.get('return', {}) - if self.process: - self.process.terminate() - salt.output.display_output( - {'local': print_ret}, - out=ret.get('out', 'nested'), - opts=self.opts, - _retcode=ret.get('retcode', salt.defaults.exitcodes.EX_OK)) - # _retcode will be available in the kwargs of the outputter function - if self.opts.get('retcode_passthrough', False): - sys.exit(ret['retcode']) - elif ret['retcode'] != salt.defaults.exitcodes.EX_OK: - sys.exit(salt.defaults.exitcodes.EX_GENERIC) - - except SaltInvocationError as err: - raise SystemExit(err) - - def _setup_caller_stack(self, opts): - ''' - Setup and return the LaneStack and Yard used by by channel when global - not already setup such as in salt-call to communicate to-from the minion - - ''' - role = opts.get('id') - if not role: - emsg = ("Missing role required to setup RAETChannel.") - log.error(emsg + "\n") - raise ValueError(emsg) - - kind = opts.get('__role') # application kind 'master', 'minion', etc - if kind not in kinds.APPL_KINDS: - emsg = ("Invalid application kind = '{0}' for RAETChannel.".format(kind)) - log.error(emsg + "\n") - raise ValueError(emsg) - if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - kinds.APPL_KIND_NAMES[kinds.applKinds.caller], ]: - lanename = "{0}_{1}".format(role, kind) - else: - emsg = ("Unsupported application kind '{0}' for RAETChannel.".format(kind)) - log.error(emsg + '\n') - raise ValueError(emsg) - - sockdirpath = opts['sock_dir'] - stackname = 'caller' + nacling.uuid(size=18) - stack = LaneStack(name=stackname, - lanename=lanename, - sockdirpath=sockdirpath) - - stack.Pk = raeting.PackKind.pack.value - stack.addRemote(RemoteYard(stack=stack, - name='manor', - lanename=lanename, - dirpath=sockdirpath)) - log.debug("Created Caller Jobber Stack %s\n", stack.name) - - return stack - - def _wait_caller(self, opts): - ''' - Returns when RAET Minion Yard is available - ''' - yardname = 'manor' - dirpath = opts['sock_dir'] - - role = opts.get('id') - if not role: - emsg = ("Missing role required to setup RAET SaltCaller.") - log.error(emsg + "\n") - raise ValueError(emsg) - - kind = opts.get('__role') # application kind 'master', 'minion', etc - if kind not in kinds.APPL_KINDS: - emsg = ("Invalid application kind = '{0}' for RAET SaltCaller.".format(kind)) - log.error(emsg + "\n") - raise ValueError(emsg) - - if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - kinds.APPL_KIND_NAMES[kinds.applKinds.caller], ]: - lanename = "{0}_{1}".format(role, kind) - else: - emsg = ("Unsupported application kind '{0}' for RAET SaltCaller.".format(kind)) - log.error(emsg + '\n') - raise ValueError(emsg) - - ha, dirpath = Yard.computeHa(dirpath, lanename, yardname) - - if is_windows(): - # RAET lanes do not use files on Windows. Need to use win32file - # API to check for existence. - exists = False - while not exists: - try: - f = win32file.CreateFile( - ha, - win32file.GENERIC_WRITE | win32file.GENERIC_READ, - win32file.FILE_SHARE_READ, - None, - win32file.OPEN_EXISTING, - 0, - None) - win32file.CloseHandle(f) - exists = True - except win32file.error: - time.sleep(0.1) - else: - while not ((os.path.exists(ha) and - not os.path.isfile(ha) and - not os.path.isdir(ha))): - time.sleep(0.1) - time.sleep(0.5) diff --git a/salt/cli/daemons.py b/salt/cli/daemons.py index 94f8304a60f..9d421016f83 100644 --- a/salt/cli/daemons.py +++ b/salt/cli/daemons.py @@ -119,7 +119,7 @@ class Master(salt.utils.parsers.MasterOptionParser, DaemonsMixin): # pylint: di Creates a master server ''' def _handle_signals(self, signum, sigframe): # pylint: disable=unused-argument - if hasattr(self.master, 'process_manager'): # IofloMaster has no process manager + if hasattr(self.master, 'process_manager'): # escalate signal to the process manager processes self.master.process_manager.stop_restarting() self.master.process_manager.send_signal_to_processes(signum) @@ -156,11 +156,6 @@ class Master(salt.utils.parsers.MasterOptionParser, DaemonsMixin): # pylint: di self.config['syndic_dir'], self.config['sqlite_queue_dir'], ] - if self.config.get('transport') == 'raet': - v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) - v_dirs.append(os.path.join(self.config['cachedir'], 'raet')) verify_env( v_dirs, self.config['user'], @@ -179,21 +174,17 @@ class Master(salt.utils.parsers.MasterOptionParser, DaemonsMixin): # pylint: di self.action_log_info('Setting up') # TODO: AIO core is separate from transport - if self.config['transport'].lower() in ('zeromq', 'tcp'): - if not verify_socket(self.config['interface'], - self.config['publish_port'], - self.config['ret_port']): - self.shutdown(4, 'The ports are not available to bind') - self.config['interface'] = ip_bracket(self.config['interface']) - migrations.migrate_paths(self.config) + if not verify_socket(self.config['interface'], + self.config['publish_port'], + self.config['ret_port']): + self.shutdown(4, 'The ports are not available to bind') + self.config['interface'] = ip_bracket(self.config['interface']) + migrations.migrate_paths(self.config) + + # Late import so logging works correctly + import salt.master + self.master = salt.master.Master(self.config) - # Late import so logging works correctly - import salt.master - self.master = salt.master.Master(self.config) - else: - # Add a udp port check here - import salt.daemons.flo - self.master = salt.daemons.flo.IofloMaster(self.config) self.daemonize_if_required() self.set_pidfile() salt.utils.process.notify_systemd() @@ -277,12 +268,6 @@ class Minion(salt.utils.parsers.MinionOptionParser, DaemonsMixin): # pylint: di confd, ] - if self.config.get('transport') == 'raet': - v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) - v_dirs.append(os.path.join(self.config['cachedir'], 'raet')) - verify_env( v_dirs, self.config['user'], @@ -318,15 +303,10 @@ class Minion(salt.utils.parsers.MinionOptionParser, DaemonsMixin): # pylint: di if self.config.get('master_type') == 'func': salt.minion.eval_master_func(self.config) self.minion = salt.minion.MinionManager(self.config) - elif transport == 'raet': - import salt.daemons.flo - self.daemonize_if_required() - self.set_pidfile() - self.minion = salt.daemons.flo.IofloMinion(self.config) else: log.error( 'The transport \'%s\' is not supported. Please use one of ' - 'the following: tcp, raet, or zeromq.', transport + 'the following: tcp, zeromq, or detect.', transport ) self.shutdown(1) @@ -384,7 +364,6 @@ class Minion(salt.utils.parsers.MinionOptionParser, DaemonsMixin): # pylint: di self.prepare() if check_user(self.config['user']): self.minion.opts['__role'] = kinds.APPL_KIND_NAMES[kinds.applKinds.caller] - self.minion.opts['raet_cleanup_protecteds'] = cleanup_protecteds self.minion.call_in() except (KeyboardInterrupt, SaltSystemExit) as exc: self.action_log_info('Stopping') @@ -469,12 +448,6 @@ class ProxyMinion(salt.utils.parsers.ProxyMinionOptionParser, DaemonsMixin): # confd, ] - if self.config.get('transport') == 'raet': - v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) - v_dirs.append(os.path.join(self.config['cachedir'], 'raet')) - verify_env( v_dirs, self.config['user'], @@ -497,24 +470,17 @@ class ProxyMinion(salt.utils.parsers.ProxyMinionOptionParser, DaemonsMixin): # self.shutdown(1) # TODO: AIO core is separate from transport - if self.config['transport'].lower() in ('zeromq', 'tcp', 'detect'): - # Late import so logging works correctly - import salt.minion - # If the minion key has not been accepted, then Salt enters a loop - # waiting for it, if we daemonize later then the minion could halt - # the boot process waiting for a key to be accepted on the master. - # This is the latest safe place to daemonize - self.daemonize_if_required() - self.set_pidfile() - if self.config.get('master_type') == 'func': - salt.minion.eval_master_func(self.config) - self.minion = salt.minion.ProxyMinionManager(self.config) - else: - # For proxy minions, this doesn't work yet. - import salt.daemons.flo - self.daemonize_if_required() - self.set_pidfile() - self.minion = salt.daemons.flo.IofloMinion(self.config) + # Late import so logging works correctly + import salt.minion + # If the minion key has not been accepted, then Salt enters a loop + # waiting for it, if we daemonize later then the minion could halt + # the boot process waiting for a key to be accepted on the master. + # This is the latest safe place to daemonize + self.daemonize_if_required() + self.set_pidfile() + if self.config.get('master_type') == 'func': + salt.minion.eval_master_func(self.config) + self.minion = salt.minion.ProxyMinionManager(self.config) def start(self): ''' @@ -543,9 +509,6 @@ class ProxyMinion(salt.utils.parsers.ProxyMinionOptionParser, DaemonsMixin): # log.error(exc) self.shutdown(exc.code) - # def call(self, cleanup_protecteds): - # This fn is omitted here, proxy minions have never supported RAET - def shutdown(self, exitcode=0, exitmsg=None): ''' If sub-classed, run any shutdown operations on this method. diff --git a/salt/cli/key.py b/salt/cli/key.py index f3824c9c9d7..6e20d6b1dee 100644 --- a/salt/cli/key.py +++ b/salt/cli/key.py @@ -17,16 +17,10 @@ class SaltKey(salt.utils.parsers.SaltKeyOptionParser): ''' import salt.key self.parse_args() - multi = False - if self.config.get('zmq_behavior') and self.config.get('transport') == 'raet': - multi = True self.setup_logfile_logger() verify_log(self.config) - if multi: - key = salt.key.MultiKeyCLI(self.config) - else: - key = salt.key.KeyCLI(self.config) + key = salt.key.KeyCLI(self.config) if check_user(self.config['user']): key.run() diff --git a/salt/client/__init__.py b/salt/client/__init__.py index 6a62e53fdc0..6e23e2372e8 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -97,16 +97,13 @@ def get_local_client( # Late import to prevent circular import import salt.config opts = salt.config.client_config(c_path) - if opts['transport'] == 'raet': - import salt.client.raet - return salt.client.raet.LocalClient(mopts=opts) + # TODO: AIO core is separate from transport - elif opts['transport'] in ('zeromq', 'tcp'): - return LocalClient( - mopts=opts, - skip_perm_errors=skip_perm_errors, - io_loop=io_loop, - auto_reconnect=auto_reconnect) + return LocalClient( + mopts=opts, + skip_perm_errors=skip_perm_errors, + io_loop=io_loop, + auto_reconnect=auto_reconnect) class LocalClient(object): diff --git a/salt/client/raet/__init__.py b/salt/client/raet/__init__.py deleted file mode 100644 index d6606e9ba02..00000000000 --- a/salt/client/raet/__init__.py +++ /dev/null @@ -1,102 +0,0 @@ -# -*- coding: utf-8 -*- -''' -The client libs to communicate with the salt master when running raet -''' -from __future__ import absolute_import, print_function, unicode_literals - -# Import python libs -import os -import time -import logging - -# Import Salt libs -import salt.config -import salt.client -import salt.utils.kinds as kinds -import salt.syspaths as syspaths - -try: - from raet import raeting, nacling - from raet.lane.stacking import LaneStack - from raet.lane.yarding import RemoteYard - HAS_RAET_LIBS = True -except ImportError: - HAS_RAET_LIBS = False - -log = logging.getLogger(__name__) - - -class LocalClient(salt.client.LocalClient): - ''' - The RAET LocalClient - ''' - def __init__(self, - c_path=os.path.join(syspaths.CONFIG_DIR, 'master'), - mopts=None): - - salt.client.LocalClient.__init__(self, c_path, mopts) - - def pub(self, - tgt, - fun, - arg=(), - tgt_type='glob', - ret='', - jid='', - timeout=5, - **kwargs): - ''' - Publish the command! - ''' - payload_kwargs = self._prep_pub( - tgt, - fun, - arg=arg, - tgt_type=tgt_type, - ret=ret, - jid=jid, - timeout=timeout, - **kwargs) - - kind = self.opts['__role'] - if kind not in kinds.APPL_KINDS: - emsg = ("Invalid application kind = '{0}' for Raet LocalClient.".format(kind)) - log.error(emsg + "\n") - raise ValueError(emsg) - if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.master], - kinds.APPL_KIND_NAMES[kinds.applKinds.syndic]]: - lanename = 'master' - else: - emsg = ("Unsupported application kind '{0}' for Raet LocalClient.".format(kind)) - log.error(emsg + '\n') - raise ValueError(emsg) - - sockdirpath = self.opts['sock_dir'] - name = 'client' + nacling.uuid(size=18) - stack = LaneStack( - name=name, - lanename=lanename, - sockdirpath=sockdirpath) - stack.Pk = raeting.PackKind.pack.value - manor_yard = RemoteYard( - stack=stack, - lanename=lanename, - name='manor', - dirpath=sockdirpath) - stack.addRemote(manor_yard) - route = {'dst': (None, manor_yard.name, 'local_cmd'), - 'src': (None, stack.local.name, None)} - msg = {'route': route, 'load': payload_kwargs} - stack.transmit(msg) - stack.serviceAll() - while True: - time.sleep(0.01) - stack.serviceAll() - while stack.rxMsgs: - msg, sender = stack.rxMsgs.popleft() - ret = msg.get('return', {}) - if 'ret' in ret: - stack.server.close() - return ret['ret'] - stack.server.close() - return ret diff --git a/salt/config/__init__.py b/salt/config/__init__.py index b3dfc8e43d4..422a5c644c7 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -960,7 +960,7 @@ VALID_OPTS = { # The size of key that should be generated when creating new keys 'keysize': int, - # The transport system for this daemon. (i.e. zeromq, raet, etc) + # The transport system for this daemon. (i.e. zeromq, tcp, detect, etc) 'transport': six.string_types, # The number of seconds to wait when the client is requesting information about running jobs @@ -1010,31 +1010,8 @@ VALID_OPTS = { 'ssh_config_file': six.string_types, 'ssh_merge_pillar': bool, - # Enable ioflo verbose logging. Warning! Very verbose! - 'ioflo_verbose': int, - - 'ioflo_period': float, - - # Set ioflo to realtime. Useful only for testing/debugging to simulate many ioflo periods very - # quickly - 'ioflo_realtime': bool, - - # Location for ioflo logs - 'ioflo_console_logdir': six.string_types, - - # The port to bind to when bringing up a RAET daemon - 'raet_port': int, - 'raet_alt_port': int, - 'raet_mutable': bool, - 'raet_main': bool, - 'raet_clear_remotes': bool, - 'raet_clear_remote_masters': bool, - 'raet_road_bufcnt': int, - 'raet_lane_bufcnt': int, 'cluster_mode': bool, - 'cluster_masters': list, 'sqlite_queue_dir': six.string_types, - 'queue_dirs': list, # Instructs the minion to ping its master(s) every n number of minutes. Used @@ -1065,9 +1042,6 @@ VALID_OPTS = { # Can be set to override the python_shell=False default in the cmd module 'cmd_safe': bool, - # Used strictly for performance testing in RAET. - 'dummy_publisher': bool, - # Used by salt-api for master requests timeout 'rest_timeout': int, @@ -1473,22 +1447,7 @@ DEFAULT_MINION_OPTS = { 'master_tops_first': False, 'auth_safemode': False, 'random_master': False, - 'minion_floscript': os.path.join(FLO_DIR, 'minion.flo'), - 'caller_floscript': os.path.join(FLO_DIR, 'caller.flo'), - 'ioflo_verbose': 0, - 'ioflo_period': 0.1, - 'ioflo_realtime': True, - 'ioflo_console_logdir': '', - 'raet_port': 4510, - 'raet_alt_port': 4511, - 'raet_mutable': False, - 'raet_main': False, - 'raet_clear_remotes': True, - 'raet_clear_remote_masters': True, - 'raet_road_bufcnt': 2, - 'raet_lane_bufcnt': 100, 'cluster_mode': False, - 'cluster_masters': [], 'restart_on_error': False, 'ping_interval': 0, 'username': None, @@ -1811,23 +1770,7 @@ DEFAULT_MASTER_OPTS = { 'ssh_identities_only': False, 'ssh_log_file': os.path.join(salt.syspaths.LOGS_DIR, 'ssh'), 'ssh_config_file': os.path.join(salt.syspaths.HOME_DIR, '.ssh', 'config'), - 'master_floscript': os.path.join(FLO_DIR, 'master.flo'), - 'worker_floscript': os.path.join(FLO_DIR, 'worker.flo'), - 'maintenance_floscript': os.path.join(FLO_DIR, 'maint.flo'), - 'ioflo_verbose': 0, - 'ioflo_period': 0.01, - 'ioflo_realtime': True, - 'ioflo_console_logdir': '', - 'raet_port': 4506, - 'raet_alt_port': 4511, - 'raet_mutable': False, - 'raet_main': True, - 'raet_clear_remotes': False, - 'raet_clear_remote_masters': True, - 'raet_road_bufcnt': 2, - 'raet_lane_bufcnt': 100, 'cluster_mode': False, - 'cluster_masters': [], 'sqlite_queue_dir': os.path.join(salt.syspaths.CACHE_DIR, 'master', 'queues'), 'queue_dirs': [], 'cli_summary': False, @@ -2125,16 +2068,6 @@ def _validate_opts(opts): if isinstance(opts.get('return'), list): opts['return'] = ','.join(opts['return']) - # RAET on Windows uses 'win32file.CreateMailslot()' for IPC. Due to this, - # sock_dirs must start with '\\.\mailslot\' and not contain any colons. - # We don't expect the user to know this, so we will fix up their path for - # them if it isn't compliant. - if (salt.utils.platform.is_windows() and opts.get('transport') == 'raet' and - 'sock_dir' in opts and - not opts['sock_dir'].startswith('\\\\.\\mailslot\\')): - opts['sock_dir'] = ( - '\\\\.\\mailslot\\' + opts['sock_dir'].replace(':', '')) - for error in errors: log.warning(error) if errors: @@ -3933,8 +3866,6 @@ def master_config(path, env_var='SALT_MASTER_CONFIG', defaults=None, exit_on_con opts['nodegroups'] = DEFAULT_MASTER_OPTS.get('nodegroups', {}) if salt.utils.data.is_dictlist(opts['nodegroups']): opts['nodegroups'] = salt.utils.data.repack_dictlist(opts['nodegroups']) - if opts.get('transport') == 'raet' and 'aes' in opts: - opts.pop('aes') apply_sdb(opts) return opts diff --git a/salt/daemons/__init__.py b/salt/daemons/__init__.py index 9d3e1ff3229..642100b6633 100644 --- a/salt/daemons/__init__.py +++ b/salt/daemons/__init__.py @@ -12,12 +12,9 @@ try: except ImportError: from collections import Iterable, Sequence, Mapping -from collections import namedtuple - import logging # Import Salt Libs -from salt.utils.odict import OrderedDict from salt.ext import six log = logging.getLogger(__name__) @@ -54,11 +51,6 @@ def extract_masters(opts, masters='master', port=None, raise_if_empty=True): By default looks for list of masters in opts['master'] and uses opts['master_port'] as the default port when otherwise not provided. - To use this function to generate the cluster master list then - call with masters='cluster_masters' and port='raet_port' on a master - and - call with masters='cluster_masters' on a minion - Use the opts key given by masters for the masters list, default is 'master' If parameter port is not None then uses the default port given by port diff --git a/salt/daemons/flo/__init__.py b/salt/daemons/flo/__init__.py deleted file mode 100644 index 80c66f68608..00000000000 --- a/salt/daemons/flo/__init__.py +++ /dev/null @@ -1,195 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Package for ioflo and raet based daemons and associated ioflo behaviors - -To use set -opts['transport'] ='raet' -master minion config -transport: raet - -See salt.config.py for relevant defaults - -opts['raet_port'] -opts['master_floscript'] -opts['minion_floscript'] -opts['ioflo_period'] -opts['ioflo_realtime'] -opts['ioflo_verbose'] -opts['caller_floscript'] -''' - -# Import Python libs -from __future__ import absolute_import, print_function, unicode_literals -import os - -# Import modules -from . import core -from . import worker -from . import maint -from . import reactor -from . import zero -from . import jobber -from . import dummy - -__all__ = ['core', 'worker', 'maint', 'zero', 'dummy', 'jobber', 'reactor'] - -# Import salt libs -import salt.daemons.masterapi -import salt.utils.stringutils -from salt.ext import six - -# Import 3rd-party libs -import ioflo.app.run # pylint: disable=3rd-party-module-not-gated - - -def explode_opts(opts): - ''' - Explode the opts into a preloads list - ''' - preloads = [(salt.utils.stringutils.to_str('.salt.opts'), dict(value=opts))] - for key, val in six.iteritems(opts): - ukey = key.replace('.', '_') - preloads.append((salt.utils.stringutils.to_str('.salt.etc.{0}'.format(ukey)), dict(value=val))) - preloads.append((salt.utils.stringutils.to_str('.salt.etc.id'), dict(value=opts['id']))) - return preloads - - -def warn_deprecated(): - salt.utils.versions.warn_until( - 'Neon', - 'The \'raet\' transport has been deprecated and will be removed in ' - 'Salt {version}. Please use \'zeromq\' or \'tcp\' transport instead.' - ) - - -class IofloMaster(object): - ''' - IofloMaster Class - ''' - def __init__(self, opts): - ''' - Assign self.opts - ''' - warn_deprecated() - self.opts = opts - self.preloads = explode_opts(self.opts) - self.access_keys = salt.daemons.masterapi.access_keys(self.opts) - self.preloads.append( - (salt.utils.stringutils.to_str('.salt.access_keys'), dict(value=self.access_keys))) - - def start(self, behaviors=None): - ''' - Start up ioflo - - port = self.opts['raet_port'] - ''' - if behaviors is None: - behaviors = [] - behaviors.extend(['salt.daemons.flo']) - - console_logdir = self.opts.get('ioflo_console_logdir', '') - if console_logdir: - consolepath = os.path.join(console_logdir, 'master.log') - else: # empty means log to std out - consolepath = '' - - ioflo.app.run.start( - name='master', - period=float(self.opts['ioflo_period']), - stamp=0.0, - real=self.opts['ioflo_realtime'], - filepath=self.opts['master_floscript'], - behaviors=behaviors, - username="", - password="", - mode=None, - houses=None, - metas=None, - preloads=self.preloads, - verbose=int(self.opts['ioflo_verbose']), - consolepath=consolepath, - ) - - -class IofloMinion(object): - ''' - IofloMinion Class - ''' - def __init__(self, opts): - ''' - Assign self.opts - ''' - warn_deprecated() - self.opts = opts - self.restart = False - - def tune_in(self, behaviors=None): - ''' - Start up ioflo - - port = self.opts['raet_port'] - ''' - if behaviors is None: - behaviors = [] - behaviors.extend(['salt.daemons.flo']) - - preloads = explode_opts(self.opts) - - console_logdir = self.opts.get('ioflo_console_logdir', '') - if console_logdir: - consolepath = os.path.join(console_logdir, 'minion.log') - else: # empty means log to std out - consolepath = '' - - ioflo.app.run.start( - name=self.opts['id'], - period=float(self.opts['ioflo_period']), - stamp=0.0, - real=self.opts['ioflo_realtime'], - filepath=self.opts['minion_floscript'], - behaviors=behaviors, - username="", - password="", - mode=None, - houses=None, - metas=None, - preloads=preloads, - verbose=int(self.opts['ioflo_verbose']), - consolepath=consolepath, - ) - - start = tune_in # alias - - def call_in(self, behaviors=None): - ''' - Start up caller minion for salt-call when there is no local minion - - ''' - if behaviors is None: - behaviors = [] - behaviors.extend(['salt.daemons.flo']) - - preloads = explode_opts(self.opts) - - console_logdir = self.opts.get('ioflo_console_logdir', '') - if console_logdir: - consolepath = os.path.join(console_logdir, 'caller.log') - else: # empty means log to std out - consolepath = '' - - ioflo.app.run.start( - name=self.opts['id'], - period=float(self.opts['ioflo_period']), - stamp=0.0, - real=self.opts['ioflo_realtime'], - filepath=self.opts['caller_floscript'], - behaviors=behaviors, - username="", - password="", - mode=None, - houses=None, - metas=None, - preloads=preloads, - verbose=int(self.opts['ioflo_verbose']), - consolepath=consolepath, - ) diff --git a/salt/daemons/flo/caller.flo b/salt/daemons/flo/caller.flo deleted file mode 100644 index 355ce50835c..00000000000 --- a/salt/daemons/flo/caller.flo +++ /dev/null @@ -1,123 +0,0 @@ -# Salt Caller floscript - -house caller - -framer callerroadstack be active first setup - # Begin the pre-flight checks - frame setup - enter - do salt raet cleanup - do salt raet road stack setup per inode ".salt.road.manor" - do salt raet manor lane setup - go loadmodules - - # Load the minion mods - frame loadmodules - do salt load modules at enter - go start - - # OK, let's start the minion up - frame start - # Start the inbound framer - bid start inbound - # Start the bootstrap framer - bid start bootstrap - # Start the eventing framer - bid start eventing - # Start the outbound framer - bid start outbound - - # Cleanup on exit - exit - do salt raet road stack closer per inode ".salt.road.manor." - do salt raet lane stack closer per inode ".salt.lane.manor." - - -framer inbound be inactive first start - frame start - do salt raet road stack service rx - do salt raet lane stack service rx - - -framer bootstrap be inactive first setup - - frame setup - enter - do salt raet road clustered per inode ".salt.road.manor." - do salt raet road usher minion setup per inode ".salt.road.manor." - go clustermaster - go multimaster - - frame clustermaster - let if salt.road.manor.cluster.clustered - print Setting Up Master Cluster .... - go join - - frame multimaster - print Setting Up Master or MultiMaster - go join - - frame join - print Joining... - enter - do salt raet road stack joiner per inode ".salt.road.manor." - recur - do salt raet road stack joined per inode ".salt.road.manor." - do salt raet road stack rejected per inode ".salt.road.manor." - - go next if joined in .salt.road.manor.status - go abort if rejected in .salt.road.manor.status - - frame joined - print Joined - go next if elapsed >= 0.5 - - frame allow - print Allowing... - enter - do salt raet road stack allower per inode ".salt.road.manor." - recur - do salt raet road stack allowed per inode ".salt.road.manor." - - go next if allowed in .salt.road.manor.status - #go abort if elapsed >= 5 - - frame allowed - print Allowed - go next if elapsed >= 0.5 - - frame clustering - print Cluster Setup ... - do salt raet road cluster load setup - go next - - frame manager - # start the manager framer - bid start manager #start alive presence from minion side - go next if elapsed >= 3.0 - - frame loading - print Loading - enter - do salt load modules - go router - - frame router - do salt raet router minion - - frame abort - bid stop all - -framer eventing be inactive first event - frame event - do salt raet eventer - -framer manager be inactive first start at 10.0 - frame start - do salt raet road stack manager per inode ".salt.road.manor" - -framer outbound be inactive first start - frame start - do salt raet lane stack service tx - do salt raet road stack service tx - diff --git a/salt/daemons/flo/core.py b/salt/daemons/flo/core.py deleted file mode 100644 index a80e5d2565c..00000000000 --- a/salt/daemons/flo/core.py +++ /dev/null @@ -1,1791 +0,0 @@ -# -*- coding: utf-8 -*- -''' -The core behaviors used by minion and master -''' -# pylint: disable=W0232 -# pylint: disable=3rd-party-module-not-gated - -# Import python libs -from __future__ import absolute_import, print_function, unicode_literals -import os -import time -import random -import logging -import itertools -from collections import deque -from _socket import gaierror - -# Import salt libs -import salt.daemons.masterapi -import salt.utils.args -import salt.utils.kinds as kinds -import salt.utils.process -import salt.utils.stringutils -import salt.transport -import salt.engines - -# pylint: disable=import-error -from raet import raeting -from raet.road.stacking import RoadStack -from raet.road.estating import RemoteEstate -from raet.lane.stacking import LaneStack -# pylint: enable=import-error - -from salt import daemons -from salt.daemons import salting -from salt.exceptions import SaltException -from salt.utils.platform import is_windows -from salt.utils.event import tagify - -# Import ioflo libs -# pylint: disable=import-error -from ioflo.aid.odicting import odict # pylint: disable=E0611,F0401 -import ioflo.base.deeding -from ioflo.base.consoling import getConsole -# pylint: enable=import-error -console = getConsole() - -# Import Third Party Libs -# pylint: disable=import-error -HAS_PSUTIL = False -try: - import salt.utils.psutil_compat as psutil - HAS_PSUTIL = True -except ImportError: - pass - -HAS_RESOURCE = False -try: - import resource - HAS_RESOURCE = True -except ImportError: - pass -# pylint: disable=no-name-in-module,redefined-builtin -from salt.ext import six -from salt.ext.six.moves import range -# pylint: enable=import-error,no-name-in-module,redefined-builtin - -log = logging.getLogger(__name__) - - -class SaltRaetCleanup(ioflo.base.deeding.Deed): - ''' - Cleanup stray lane keep directories not reaped - - FloScript: - - do salt raet cleanup at enter - - ''' - Ioinits = { - 'opts': salt.utils.stringutils.to_str('.salt.opts'), - } - - def action(self): - ''' - Should only run once to cleanup stale lane uxd files. - ''' - if not is_windows() and self.opts.value.get('sock_dir'): - sockdirpath = os.path.abspath(self.opts.value['sock_dir']) - console.concise("Cleaning up uxd files in {0}\n".format(sockdirpath)) - protecteds = self.opts.value.get('raet_cleanup_protecteds', []) - for name in os.listdir(sockdirpath): - path = os.path.join(sockdirpath, name) - if os.path.isdir(path): - continue - root, ext = os.path.splitext(name) - if ext != '.uxd': - continue - if not all(root.partition('.')): - continue - if path in protecteds: - continue - try: - os.unlink(path) - console.concise("Removed {0}\n".format(path)) - except OSError: - console.concise("Failed removing {0}\n".format(path)) - raise - - -class SaltRaetRoadClustered(ioflo.base.deeding.Deed): - ''' - Updates value of share .salt.road.manor.cluster.clustered - Twith opts['cluster_mode'] - - FloScript: - - do salt raet road clustered - go next if .salt.road.manor.cluster.clustered - - ''' - Ioinits = odict(inode=salt.utils.stringutils.to_str('.salt.road.manor.'), - clustered=odict(ipath=salt.utils.stringutils.to_str('cluster.clustered'), - ival=False), - opts=salt.utils.stringutils.to_str('.salt.opts'),) - - def action(self, **kwa): - ''' - Update .cluster.clustered share from opts - ''' - self.clustered.update(value=self.opts.value.get('cluster_mode', False)) - - -class SaltRaetProcessManagerSetup(ioflo.base.deeding.Deed): - ''' - Set up the process manager object - ''' - Ioinits = {'proc_mgr': salt.utils.stringutils.to_str('.salt.usr.proc_mgr')} - - def action(self): - ''' - Create the process manager - ''' - self.proc_mgr.value = salt.utils.process.ProcessManager() - - -class SaltRaetRoadUsherMinionSetup(ioflo.base.deeding.Deed): - ''' - Set up .ushers which is initial list of masters to bootstrap - into road - - FloScript: - - do salt raet road usher minion setup at enter - - ''' - Ioinits = odict( - inode=salt.utils.stringutils.to_str('.salt.road.manor.'), - ushers=salt.utils.stringutils.to_str('ushers'), - opts=salt.utils.stringutils.to_str('.salt.opts')) - - def action(self): - ''' - Assign .ushers by parsing opts - ''' - masters = 'master' - port = None - if self.opts.value.get('cluster_mode', False): - masters = 'cluster_masters' - - self.ushers.value = daemons.extract_masters(self.opts.value, - masters=masters, - port=port) - - -class SaltRaetRoadUsherMasterSetup(ioflo.base.deeding.Deed): - ''' - Set up .ushers which is initial list of masters to bootstrap - into road - - FloScript: - - do salt raet road usher master setup at enter - - ''' - Ioinits = odict( - inode=salt.utils.stringutils.to_str('.salt.road.manor.'), - ushers=salt.utils.stringutils.to_str('ushers'), - opts=salt.utils.stringutils.to_str('.salt.opts')) - - def action(self): - ''' - Assign .ushers by parsing opts - ''' - masters = 'cluster_masters' - port = 'raet_port' - - self.ushers.value = daemons.extract_masters(self.opts.value, - masters=masters, - port=port, - raise_if_empty=False) - - -class SaltRaetRoadClusterLoadSetup(ioflo.base.deeding.Deed): - ''' - Sets up cluster.masters for load balancing - - FloScript: - - do salt raet road cluster load setup at enter - - ''' - Ioinits = odict( - inode=salt.utils.stringutils.to_str('.salt.road.manor.'), - masters={'ipath': salt.utils.stringutils.to_str('cluster.masters'), 'ival': odict()}, - stack=salt.utils.stringutils.to_str('stack'), - opts=salt.utils.stringutils.to_str('.salt.opts'),) - - def action(self, **kwa): - ''' - Populate loads from masters in stack.remotes - ''' - if self.opts.value.get('cluster_mode'): - for remote in list(self.stack.value.remotes.values()): - if remote.kind == kinds.applKinds.master: - self.masters.value[remote.name] = odict(load=0.0, expire=self.store.stamp) - - -class SaltRaetRoadStackSetup(ioflo.base.deeding.Deed): - ''' - Initialize and run raet udp stack for Salt - FloScript: - - do salt raet road stack setup at enter - - ''' - Ioinits = { - 'inode': salt.utils.stringutils.to_str('salt.road.manor.'), - 'stack': salt.utils.stringutils.to_str('stack'), - 'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'txmsgs': {'ipath': salt.utils.stringutils.to_str('txmsgs'), - 'ival': deque()}, - 'rxmsgs': {'ipath': salt.utils.stringutils.to_str('rxmsgs'), - 'ival': deque()}, - 'local': {'ipath': salt.utils.stringutils.to_str('local'), - 'ival': {'main': False, - 'mutable': False, - 'uid': None, - 'role': 'master', - 'sighex': None, - 'prihex': None, - 'bufcnt': 2}}, - } - - def _prepare(self): - ''' - Assign class defaults - ''' - RoadStack.Bk = raeting.BodyKind.msgpack.value - RoadStack.JoinentTimeout = 0.0 - - def action(self): - ''' - enter action - should only run once to setup road stack. - moved from _prepare so can do clean up before stack is initialized - - do salt raet road stack setup at enter - ''' - kind = self.opts.value['__role'] # application kind - if kind not in kinds.APPL_KINDS: - emsg = ("Invalid application kind = '{0}'.".format(kind)) - log.error(emsg + '\n') - raise ValueError(emsg) - role = self.opts.value.get('id', '') - if not role: - emsg = ("Missing role required to setup RoadStack.") - log.error(emsg + "\n") - raise ValueError(emsg) - - name = "{0}_{1}".format(role, kind) - main = self.opts.value.get('raet_main', self.local.data.main) - mutable = self.opts.value.get('raet_mutable', self.local.data.mutable) - always = self.opts.value.get('open_mode', False) - mutable = mutable or always # open_made when True takes precedence - uid = self.local.data.uid - - if kind == kinds.APPL_KIND_NAMES[kinds.applKinds.caller]: - ha = (self.opts.value['interface'], self.opts.value['raet_alt_port']) - else: - ha = (self.opts.value['interface'], self.opts.value['raet_port']) - - basedirpath = os.path.abspath(os.path.join(self.opts.value['cachedir'], 'raet')) - - txMsgs = self.txmsgs.value - rxMsgs = self.rxmsgs.value - - keep = salting.SaltKeep(opts=self.opts.value, - basedirpath=basedirpath, - stackname=name) - - roledata = keep.loadLocalRoleData() - sighex = roledata['sighex'] or self.local.data.sighex - prihex = roledata['prihex'] or self.local.data.prihex - - bufcnt = self.opts.value.get('raet_road_bufcnt', self.local.data.bufcnt) - - self.stack.value = RoadStack(store=self.store, - keep=keep, - name=name, - uid=uid, - ha=ha, - role=role, - sigkey=sighex, - prikey=prihex, - main=main, - kind=kinds.APPL_KINDS[kind], - mutable=mutable, - txMsgs=txMsgs, - rxMsgs=rxMsgs, - period=3.0, - offset=0.5, - bufcnt=bufcnt) - - if self.opts.value.get('raet_clear_remotes'): - for remote in list(self.stack.value.remotes.values()): - self.stack.value.removeRemote(remote, clear=True) - self.stack.puid = self.stack.value.Uid # reset puid - - -class SaltRaetRoadStackCloser(ioflo.base.deeding.Deed): - ''' - Closes stack server socket connection - FloScript: - - do salt raet road stack closer at exit - - ''' - Ioinits = odict( - inode=salt.utils.stringutils.to_str('.salt.road.manor.'), - stack=salt.utils.stringutils.to_str('stack'), ) - - def action(self, **kwa): - ''' - Close udp socket - ''' - if self.stack.value and isinstance(self.stack.value, RoadStack): - self.stack.value.server.close() - - -class SaltRaetRoadStackJoiner(ioflo.base.deeding.Deed): - ''' - Initiates join transaction with master(s) - FloScript: - - do salt raet road stack joiner at enter - - assumes that prior the following has been run to setup .masters - - do salt raet road usher minion setup - - ''' - Ioinits = odict( - inode=salt.utils.stringutils.to_str('.salt.road.manor.'), - stack=salt.utils.stringutils.to_str('stack'), - ushers=salt.utils.stringutils.to_str('ushers'), - opts=salt.utils.stringutils.to_str('.salt.opts')) - - def action(self, **kwa): - ''' - Join with all masters - ''' - stack = self.stack.value - if stack and isinstance(stack, RoadStack): - refresh_masters = (self.opts.value.get('raet_clear_remote_masters', - True) or not stack.remotes) - - refresh_all = (self.opts.value.get('raet_clear_remotes', True) or - not stack.remotes) - - if refresh_masters: # clear all remote masters - for remote in list(stack.remotes.values()): - if remote.kind == kinds.applKinds.master: - stack.removeRemote(remote, clear=True) - - if refresh_all: # clear all remotes - for remote in list(stack.remotes.values()): - stack.removeRemote(remote, clear=True) - - if refresh_all or refresh_masters: - stack.puid = stack.Uid # reset puid so reuse same uid each time - - ex = SaltException('Unable to connect to any master') - for master in self.ushers.value: - try: - mha = master['external'] - stack.addRemote(RemoteEstate(stack=stack, - fuid=0, # vacuous join - sid=0, # always 0 for join - ha=mha, - kind=kinds.applKinds.master)) - except gaierror as ex: - log.warning("Unable to connect to master %s: %s", mha, ex) - if self.opts.value.get('master_type') not in ('failover', 'distributed'): - raise ex - if not stack.remotes: - raise ex - - for remote in list(stack.remotes.values()): - if remote.kind == kinds.applKinds.master: - stack.join(uid=remote.uid, timeout=0.0) - - -class SaltRaetRoadStackJoined(ioflo.base.deeding.Deed): - ''' - Updates status with .joined of zeroth remote estate (master) - FloScript: - - do salt raet road stack joined - go next if joined in .salt.road.manor.status - - ''' - Ioinits = odict( - inode=salt.utils.stringutils.to_str('.salt.road.manor.'), - stack=salt.utils.stringutils.to_str('stack'), - status=odict(ipath=salt.utils.stringutils.to_str('status'), - ival=odict(joined=False, - allowed=False, - alived=False, - rejected=False, - idle=False, ))) - - def action(self, **kwa): - ''' - Update .status share - ''' - stack = self.stack.value - joined = False - if stack and isinstance(stack, RoadStack): - if stack.remotes: - joined = any([remote.joined for remote in list(stack.remotes.values()) - if remote.kind == kinds.applKinds.master]) - self.status.update(joined=joined) - - -class SaltRaetRoadStackRejected(ioflo.base.deeding.Deed): - ''' - Updates status with rejected of .acceptance of zeroth remote estate (master) - FloScript: - - do salt raet road stack rejected - go next if rejected in .salt.road.manor.status - - ''' - Ioinits = odict( - inode=salt.utils.stringutils.to_str('.salt.road.manor.'), - stack=salt.utils.stringutils.to_str('stack'), - status=odict(ipath=salt.utils.stringutils.to_str('status'), - ival=odict(joined=False, - allowed=False, - alived=False, - rejected=False, - idle=False, ))) - - def action(self, **kwa): - ''' - Update .status share - ''' - stack = self.stack.value - rejected = False - if stack and isinstance(stack, RoadStack): - if stack.remotes: - rejected = all([remote.acceptance == raeting.Acceptance.rejected.value - for remote in stack.remotes.values() - if remote.kind == kinds.applKinds.master]) - else: # no remotes so assume rejected - rejected = True - self.status.update(rejected=rejected) - - -class SaltRaetRoadStackAllower(ioflo.base.deeding.Deed): - ''' - Initiates allow (CurveCP handshake) transaction with master - FloScript: - - do salt raet road stack allower at enter - - ''' - Ioinits = odict( - inode=salt.utils.stringutils.to_str('.salt.road.manor.'), - stack=salt.utils.stringutils.to_str('stack'), ) - - def action(self, **kwa): - ''' - Receive any udp packets on server socket and put in rxes - Send any packets in txes - ''' - stack = self.stack.value - if stack and isinstance(stack, RoadStack): - for remote in stack.remotes.values(): - if remote.kind == kinds.applKinds.master: - stack.allow(uid=remote.uid, timeout=0.0) - - -class SaltRaetRoadStackAllowed(ioflo.base.deeding.Deed): - ''' - Updates status with .allowed of zeroth remote estate (master) - FloScript: - - do salt raet road stack allowed - go next if allowed in .salt.road.manor.status - - ''' - Ioinits = odict( - inode=salt.utils.stringutils.to_str('.salt.road.manor.'), - stack=salt.utils.stringutils.to_str('stack'), - status=odict(ipath=salt.utils.stringutils.to_str('status'), - ival=odict(joined=False, - allowed=False, - alived=False, - rejected=False, - idle=False, ))) - - def action(self, **kwa): - ''' - Update .status share - ''' - stack = self.stack.value - allowed = False - if stack and isinstance(stack, RoadStack): - if stack.remotes: - allowed = any([remote.allowed for remote in list(stack.remotes.values()) - if remote.kind == kinds.applKinds.master]) - self.status.update(allowed=allowed) - - -class SaltRaetRoadStackManager(ioflo.base.deeding.Deed): - ''' - Runs the manage method of RoadStack - FloScript: - do salt raet road stack manager - - ''' - Ioinits = odict( - inode=salt.utils.stringutils.to_str('.salt.road.manor.'), - stack=salt.utils.stringutils.to_str('stack'), - alloweds={'ipath': salt.utils.stringutils.to_str('.salt.var.presence.alloweds'), - 'ival': odict()}, - aliveds={'ipath': salt.utils.stringutils.to_str('.salt.var.presence.aliveds'), - 'ival': odict()}, - reapeds={'ipath': salt.utils.stringutils.to_str('.salt.var.presence.reapeds'), - 'ival': odict()}, - availables={'ipath': salt.utils.stringutils.to_str('.salt.var.presence.availables'), - 'ival': set()}, - changeds={'ipath': salt.utils.stringutils.to_str('.salt.var.presence.changeds'), - 'ival': odict(plus=set(), minus=set())}, - event=salt.utils.stringutils.to_str('.salt.event.events'),) - - def _fire_events(self): - stack = self.stack.value - if self.changeds.data.plus or self.changeds.data.minus: - # fire presence change event - data = {'new': list(self.changeds.data.plus), - 'lost': list(self.changeds.data.minus)} - tag = tagify('change', 'presence') - route = {'dst': (None, None, 'event_fire'), - 'src': (None, stack.local.name, None)} - msg = {'route': route, 'tag': tag, 'data': data} - self.event.value.append(msg) - # fire presence present event - data = {'present': list(self.aliveds.value)} - tag = tagify('present', 'presence') - route = {'dst': (None, None, 'event_fire'), - 'src': (None, stack.local.name, None)} - msg = {'route': route, 'tag': tag, 'data': data} - self.event.value.append(msg) - - def action(self, **kwa): - ''' - Manage the presence of any remotes - - availables is set of names of alive remotes which are also allowed - changeds is is share with two fields: - plus is set of names of newly available remotes - minus is set of names of newly unavailable remotes - alloweds is dict of allowed remotes keyed by name - aliveds is dict of alived remotes keyed by name - reapeds is dict of reaped remotes keyed by name - ''' - stack = self.stack.value - if stack and isinstance(stack, RoadStack): - stack.manage(cascade=True) - # make copies - self.availables.value = set(self.stack.value.availables) - self.changeds.update(plus=set(self.stack.value.changeds['plus'])) - self.changeds.update(minus=set(self.stack.value.changeds['minus'])) - self.alloweds.value = odict(self.stack.value.alloweds) - self.aliveds.value = odict(self.stack.value.aliveds) - self.reapeds.value = odict(self.stack.value.reapeds) - - console.concise(" Manage {0}.\nAvailables: {1}\nChangeds:\nPlus: {2}\n" - "Minus: {3}\nAlloweds: {4}\nAliveds: {5}\nReapeds: {6}\n".format( - stack.name, - self.availables.value, - self.changeds.data.plus, - self.changeds.data.minus, - self.alloweds.value, - self.aliveds.value, - self.reapeds.value)) - - self._fire_events() - - -class SaltRaetRoadStackPrinter(ioflo.base.deeding.Deed): - ''' - Prints out messages on rxMsgs queue for associated stack - FloScript: - - do raet road stack printer - - ''' - Ioinits = odict( - inode=salt.utils.stringutils.to_str('.salt.road.manor.'), - rxmsgs=odict(ipath=salt.utils.stringutils.to_str('rxmsgs'), ival=deque()),) - - def action(self, **kwa): - ''' - Queue up message - ''' - rxMsgs = self.rxmsgs.value - while rxMsgs: - msg, name = rxMsgs.popleft() - console.terse("\nReceived....\n{0}\n".format(msg)) - - -class SaltLoadModules(ioflo.base.deeding.Deed): - ''' - Reload the minion modules - FloScript: - - do salt load modules at enter - - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'grains': salt.utils.stringutils.to_str('.salt.grains'), - 'utils': salt.utils.stringutils.to_str('.salt.loader.utils'), - 'modules': salt.utils.stringutils.to_str('.salt.loader.modules'), - 'grain_time': salt.utils.stringutils.to_str('.salt.var.grain_time'), - 'module_refresh': salt.utils.stringutils.to_str('.salt.var.module_refresh'), - 'returners': salt.utils.stringutils.to_str('.salt.loader.returners'), - 'module_executors': salt.utils.stringutils.to_str('.salt.loader.executors')} - - def _prepare(self): - self._load_modules() - - def action(self): - self._load_modules() - - def _load_modules(self): - ''' - Return the functions and the returners loaded up from the loader - module - ''' - if self.grain_time.value is None: - self.grain_time.value = 0.0 - # if this is a *nix system AND modules_max_memory is set, lets enforce - # a memory limit on module imports - # this feature ONLY works on *nix like OSs (resource module doesn't work on windows) - modules_max_memory = False - if self.opts.value.get('modules_max_memory', -1) > 0 and HAS_PSUTIL and HAS_RESOURCE: - log.debug('modules_max_memory set, enforcing a maximum of %s', - self.opts.value['modules_max_memory']) - modules_max_memory = True - old_mem_limit = resource.getrlimit(resource.RLIMIT_AS) - rss, vms = psutil.Process(os.getpid()).memory_info()[:2] - mem_limit = rss + vms + self.opts.value['modules_max_memory'] - resource.setrlimit(resource.RLIMIT_AS, (mem_limit, mem_limit)) - elif self.opts.value.get('modules_max_memory', -1) > 0: - if not HAS_PSUTIL: - log.error('Unable to enforce modules_max_memory because psutil is missing') - if not HAS_RESOURCE: - log.error('Unable to enforce modules_max_memory because resource is missing') - - if time.time() - self.grain_time.value > 300.0 or self.module_refresh.value: - self.opts.value['grains'] = salt.loader.grains(self.opts.value) - self.grain_time.value = time.time() - self.grains.value = self.opts.value['grains'] - self.utils.value = salt.loader.utils(self.opts.value) - self.modules.value = salt.loader.minion_mods(self.opts.value, utils=self.utils.value) - self.returners.value = salt.loader.returners(self.opts.value, self.modules.value) - self.module_executors.value = salt.loader.executors(self.opts.value, self.modules.value) - - self.utils.value.clear() - self.modules.value.clear() - self.returners.value.clear() - self.module_executors.value.clear() - - # we're done, reset the limits! - if modules_max_memory is True: - resource.setrlimit(resource.RLIMIT_AS, old_mem_limit) - self.module_refresh.value = False - - -class SaltLoadPillar(ioflo.base.deeding.Deed): - ''' - Load up the initial pillar for the minion - - do salt load pillar - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'pillar': salt.utils.stringutils.to_str('.salt.pillar'), - 'grains': salt.utils.stringutils.to_str('.salt.grains'), - 'modules': salt.utils.stringutils.to_str('.salt.loader.modules'), - 'pillar_refresh': salt.utils.stringutils.to_str('.salt.var.pillar_refresh'), - 'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'), - 'master_estate_name': salt.utils.stringutils.to_str('.salt.track.master_estate_name') - } - - def action(self): - ''' - Initial pillar - ''' - # default master is the first remote that is allowed - available_masters = [remote for remote in list(self.road_stack.value.remotes.values()) - if remote.allowed] - while not available_masters: - available_masters = [remote for remote in self.road_stack.value.remotes.values() - if remote.allowed] - time.sleep(0.1) - - random_master = self.opts.value.get('random_master') - if random_master: - master = available_masters[random.randint(0, len(available_masters) - 1)] - else: - master = available_masters[0] - - self.master_estate_name.value = master.name - - route = {'src': (self.road_stack.value.local.name, None, None), - 'dst': (master.name, None, 'remote_cmd')} - load = {'id': self.opts.value['id'], - 'grains': self.grains.value, - 'saltenv': self.opts.value['saltenv'], - 'ver': '2', - 'cmd': '_pillar'} - self.road_stack.value.transmit({'route': route, 'load': load}, - uid=master.uid) - self.road_stack.value.serviceAll() - while True: - time.sleep(0.1) - while self.road_stack.value.rxMsgs: - msg, sender = self.road_stack.value.rxMsgs.popleft() - self.pillar.value = msg.get('return', {}) - if self.pillar.value is None: - continue - self.opts.value['pillar'] = self.pillar.value - self.pillar_refresh.value = False - return - self.road_stack.value.serviceAll() - - -class SaltSchedule(ioflo.base.deeding.Deed): - ''' - Evaluates the schedule - FloScript: - - do salt schedule - - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'grains': salt.utils.stringutils.to_str('.salt.grains'), - 'utils': salt.utils.stringutils.to_str('.salt.loader.utils'), - 'modules': salt.utils.stringutils.to_str('.salt.loader.modules'), - 'returners': salt.utils.stringutils.to_str('.salt.loader.returners')} - - def _prepare(self): - ''' - Map opts and make the schedule object - ''' - self.utils.value = salt.loader.utils(self.opts.value) - self.modules.value = salt.loader.minion_mods(self.opts.value, utils=self.utils.value) - self.returners.value = salt.loader.returners(self.opts.value, self.modules.value) - self.schedule = salt.utils.schedule.Schedule( - self.opts.value, - self.modules.value, - self.returners.value) - - def action(self): - ''' - Eval the schedule - ''' - self.schedule.eval() - - -class SaltRaetManorLaneSetup(ioflo.base.deeding.Deed): - ''' - Only intended to be called once at the top of the manor house - Sets up the LaneStack for the main yard - FloScript: - - do salt raet manor lane setup at enter - - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'event_yards': salt.utils.stringutils.to_str('.salt.event.yards'), - 'local_cmd': salt.utils.stringutils.to_str('.salt.var.local_cmd'), - 'remote_cmd': salt.utils.stringutils.to_str('.salt.var.remote_cmd'), - 'publish': salt.utils.stringutils.to_str('.salt.var.publish'), - 'fun': salt.utils.stringutils.to_str('.salt.var.fun'), - 'worker_verify': salt.utils.stringutils.to_str('.salt.var.worker_verify'), - 'event': salt.utils.stringutils.to_str('.salt.event.events'), - 'event_req': salt.utils.stringutils.to_str('.salt.event.event_req'), - 'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), - 'stats_req': salt.utils.stringutils.to_str('.salt.stats.event_req'), - 'workers': salt.utils.stringutils.to_str('.salt.track.workers'), - 'inode': salt.utils.stringutils.to_str('.salt.lane.manor.'), - 'stack': salt.utils.stringutils.to_str('stack'), - 'local': {'ipath': salt.utils.stringutils.to_str('local'), - 'ival': {'lanename': 'master', - 'bufcnt': 100}}, - } - - def _prepare(self): - ''' - Set up required objects and queues - ''' - pass - - def action(self): - ''' - Run once at enter - ''' - kind = self.opts.value['__role'] - if kind not in kinds.APPL_KINDS: - emsg = ("Invalid application kind = '{0}' for manor lane.".format(kind)) - log.error(emsg + "\n") - raise ValueError(emsg) - - if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.master], - kinds.APPL_KIND_NAMES[kinds.applKinds.syndic]]: - lanename = 'master' - elif kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - kinds.APPL_KIND_NAMES[kinds.applKinds.caller], ]: - role = self.opts.value.get('id', '') - if not role: - emsg = ("Missing role required to setup manor Lane.") - log.error(emsg + "\n") - raise ValueError(emsg) - lanename = "{0}_{1}".format(role, kind) - else: - emsg = ("Unsupported application kind = '{0}' for manor Lane.".format(kind)) - log.error(emsg + '\n') - raise ValueError(emsg) - - bufcnt = self.opts.value.get('raet_lane_bufcnt', self.local.data.bufcnt) - - name = 'manor' - self.stack.value = LaneStack( - name=name, - lanename=lanename, - sockdirpath=self.opts.value['sock_dir'], - bufcnt=bufcnt) - self.stack.value.Pk = raeting.PackKind.pack.value - self.event_yards.value = set() - self.local_cmd.value = deque() - self.remote_cmd.value = deque() - self.fun.value = deque() - self.event.value = deque() - self.event_req.value = deque() - self.presence_req.value = deque() - self.stats_req.value = deque() - self.publish.value = deque() - self.worker_verify.value = salt.utils.stringutils.random() - if self.opts.value.get('worker_threads'): - worker_seed = [] - for index in range(self.opts.value['worker_threads']): - worker_seed.append('worker{0}'.format(index + 1)) - self.workers.value = itertools.cycle(worker_seed) - - -class SaltRaetLaneStackCloser(ioflo.base.deeding.Deed): # pylint: disable=W0232 - ''' - Closes lane stack server socket connection - FloScript: - - do raet lane stack closer at exit - - ''' - Ioinits = odict( - inode='.salt.lane.manor', - stack='stack',) - - def action(self, **kwa): - ''' - Close uxd socket - ''' - if self.stack.value and isinstance(self.stack.value, LaneStack): - self.stack.value.server.close() - - -class SaltRaetRoadStackService(ioflo.base.deeding.Deed): - ''' - Process the udp traffic - FloScript: - - do rx - - ''' - Ioinits = { - 'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'), - } - - def action(self): - ''' - Process inboud queues - ''' - self.road_stack.value.serviceAll() - - -class SaltRaetRoadStackServiceRx(ioflo.base.deeding.Deed): - ''' - Process the inbound Road traffic - FloScript: - - do salt raet road stack service rx - - ''' - Ioinits = { - 'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'), - } - - def action(self): - ''' - Process inboud queues - ''' - self.road_stack.value.serviceAllRx() - - -class SaltRaetRoadStackServiceTx(ioflo.base.deeding.Deed): - ''' - Process the outbound Road traffic - FloScript: - - do salt raet road stack service tx - - ''' - # Yes, this class is identical to RX, this is because we still need to - # separate out rx and tx in raet itself - Ioinits = { - 'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'), - } - - def action(self): - ''' - Process inbound queues - ''' - self.road_stack.value.serviceAllTx() - - -class SaltRaetLaneStackServiceRx(ioflo.base.deeding.Deed): - ''' - Process the inbound Lane traffic - FloScript: - - do salt raet lane stack service rx - - ''' - Ioinits = { - 'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'), - } - - def action(self): - ''' - Process inboud queues - ''' - self.lane_stack.value.serviceAllRx() - - -class SaltRaetLaneStackServiceTx(ioflo.base.deeding.Deed): - ''' - Process the outbound Lane traffic - FloScript: - - do salt raet lane stack service tx - - ''' - # Yes, this class is identical to RX, this is because we still need to - # separate out rx and tx in raet itself - Ioinits = { - 'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'), - } - - def action(self): - ''' - Process outbound queues - ''' - self.lane_stack.value.serviceAllTx() - - -class SaltRaetRouter(ioflo.base.deeding.Deed): - ''' - Routes the communication in and out of Road and Lane connections - - This is a base class - - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'local_cmd': salt.utils.stringutils.to_str('.salt.var.local_cmd'), - 'remote_cmd': salt.utils.stringutils.to_str('.salt.var.remote_cmd'), - 'publish': salt.utils.stringutils.to_str('.salt.var.publish'), - 'fun': salt.utils.stringutils.to_str('.salt.var.fun'), - 'event': salt.utils.stringutils.to_str('.salt.event.events'), - 'event_req': salt.utils.stringutils.to_str('.salt.event.event_req'), # deque - 'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), # deque - 'stats_req': salt.utils.stringutils.to_str('.salt.stats.event_req'), # deque - 'availables': salt.utils.stringutils.to_str('.salt.var.presence.availables'), # set() - 'workers': salt.utils.stringutils.to_str('.salt.track.workers'), - 'worker_verify': salt.utils.stringutils.to_str('.salt.var.worker_verify'), - 'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'), - 'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'), - 'master_estate_name': salt.utils.stringutils.to_str('.salt.track.master_estate_name'), - # requeuing when not yet routable - 'laters': {'ipath': salt.utils.stringutils.to_str('.salt.lane.manor.laters'), - 'ival': deque()}} - - def _process_road_rxmsg(self, msg, sender): - ''' - Send to the right queue - msg is the message body dict - sender is the unique name of the remote estate that sent the message - ''' - pass - - def _process_lane_rxmsg(self, msg, sender): - ''' - Send uxd messages tot he right queue or forward them to the correct - yard etc. - - msg is message body dict - sender is unique name of remote that sent the message - ''' - pass - - def _get_master_estate_name(self, clustered=False): - ''' - Assign and return the name of the estate for the default master or empty if none - If the default master is no longer available then selects one of the available - masters - - If clustered is True then use load balancing algorithm to select master - ''' - opts = self.opts.value - master = self.road_stack.value.nameRemotes.get(self.master_estate_name.value) - if not master or not master.alived: # select a different master - available_masters = [remote for remote in - six.Iterator(self.road_stack.value.remotes) - if remote.alived] - if available_masters: - random_master = opts.get('random_master') - if random_master: - master = available_masters[random.randint(0, len(available_masters) - 1)] - else: - master = available_masters[0] - else: - master = None - - self.master_estate_name.value = master.name if master else '' - - return self.master_estate_name.value - - def _availablize(self, minions): - ''' - Return set that is intersection of associated minion estates for - roles in minions and the set of available minion estates. - ''' - suffix = '_{0}'.format(kinds.APPL_KIND_NAMES[kinds.applKinds.minion]) - return list(set(minions) & - set((name.rstrip(suffix) for name in self.availables.value))) - - def action(self): - ''' - Process the messages! - ''' - while self.road_stack.value.rxMsgs: - msg, sender = self.road_stack.value.rxMsgs.popleft() - self._process_road_rxmsg(msg=msg, sender=sender) - while self.laters.value: # process requeued LaneMsgs - msg, sender = self.laters.value.popleft() - self.lane_stack.value.rxMsgs.append((msg, sender)) - while self.lane_stack.value.rxMsgs: - msg, sender = self.lane_stack.value.rxMsgs.popleft() - self._process_lane_rxmsg(msg=msg, sender=sender) - - -class SaltRaetRouterMaster(SaltRaetRouter): - ''' - Routes the communication in and out of Road and Lane connections - Specific to Master - - do salt raet router master - - ''' - def _process_road_rxmsg(self, msg, sender): - ''' - Send to the right queue - msg is the message body dict - sender is the unique name of the remote estate that sent the message - ''' - try: - s_estate, s_yard, s_share = msg['route']['src'] - d_estate, d_yard, d_share = msg['route']['dst'] - except (ValueError, IndexError): - log.error('Received invalid message: %s', msg) - return - - if s_estate is None: # drop - return - - log.debug( - '**** Road Router rxMsg **** id=%s estate=%s yard=%s\nmsg=%s', - self.opts.value['id'], - self.road_stack.value.local.name, - self.lane_stack.value.local.name, - msg - ) - - if d_estate is not None and d_estate != self.road_stack.value.local.name: - log.error('Road Router Received message for wrong estate: %s', d_estate) - return - - if d_yard is not None: - # Meant for another yard, send it off! - if d_yard in self.lane_stack.value.nameRemotes: - self.lane_stack.value.transmit(msg, - self.lane_stack.value.nameRemotes[d_yard].uid) - return - if d_share is None: - # No queue destination! - log.error('Received message without share: %s', msg) - return - elif d_share == 'event_fire': # rebroadcast events from other masters - self.event.value.append(msg) - #log.debug("\n**** Event Fire \n %s\n", msg) - return - elif d_share == 'local_cmd': - # Refuse local commands over the wire - log.error('Received local command remotely! Ignoring: %s', msg) - return - elif d_share == 'remote_cmd': - # Send it to a remote worker - if 'load' in msg: - role = self.road_stack.value.nameRemotes[sender].role - msg['load']['id'] = role # sender # should this be role XXXX - self.lane_stack.value.transmit(msg, - self.lane_stack.value.fetchUidByName(next(self.workers.value))) - - def _process_lane_rxmsg(self, msg, sender): - ''' - Send uxd messages tot he right queue or forward them to the correct - yard etc. - - msg is message body dict - sender is unique name of remote that sent the message - ''' - try: - s_estate, s_yard, s_share = msg['route']['src'] - d_estate, d_yard, d_share = msg['route']['dst'] - except (ValueError, IndexError): - log.error('Lane Router Received invalid message: %s', msg) - return - - if s_yard is None: - return # drop message - - if s_estate is None: # substitute local estate - s_estate = self.road_stack.value.local.name - msg['route']['src'] = (s_estate, s_yard, s_share) - - log.debug( - '**** Lane Router rxMsg **** id=%s estate=%s yard=%s\nmsg=%s', - self.opts.value['id'], - self.road_stack.value.local.name, - self.lane_stack.value.local.name, - msg - ) - - if d_estate is None: - pass - elif d_estate != self.road_stack.value.local.name: - # Forward to the correct estate - if d_estate in self.road_stack.value.nameRemotes: - self.road_stack.value.message(msg, - self.road_stack.value.nameRemotes[d_estate].uid) - return - - if d_share == 'pub_ret': - # only publish to available minions - msg['return']['ret']['minions'] = self._availablize(msg['return']['ret']['minions']) - if msg.get('__worker_verify') == self.worker_verify.value: - self.publish.value.append(msg) - - if d_yard is None: - pass - elif d_yard != self.lane_stack.value.local.name: - # Meant for another yard, send it off! - if d_yard in self.lane_stack.value.nameRemotes: - self.lane_stack.value.transmit(msg, - self.lane_stack.value.nameRemotes[d_yard].uid) - return - if d_share is None: - # No queue destination! - log.error('Lane Router Received message without share: %s', msg) - return - elif d_share == 'local_cmd': - self.lane_stack.value.transmit(msg, - self.lane_stack.value.fetchUidByName(next(self.workers.value))) - elif d_share == 'event_req': - self.event_req.value.append(msg) - #log.debug("\n**** Event Subscribe \n %s\n", msg) - elif d_share == 'event_fire': - self.event.value.append(msg) - #log.debug("\n**** Event Fire \n %s\n", msg) - elif d_share == 'presence_req': - self.presence_req.value.append(msg) - #log.debug("\n**** Presence Request \n %s\n", msg) - elif d_share == 'stats_req': - self.stats_req.value.append(msg) - #log.debug("\n**** Stats Request \n %s\n", msg) - - -class SaltRaetRouterMinion(SaltRaetRouter): - ''' - Routes the communication in and out of Road and Lane connections - Specific to Minions - - do salt raet router minion - - ''' - def _process_road_rxmsg(self, msg, sender): - ''' - Send to the right queue - msg is the message body dict - sender is the unique name of the remote estate that sent the message - ''' - try: - s_estate, s_yard, s_share = msg['route']['src'] - d_estate, d_yard, d_share = msg['route']['dst'] - except (ValueError, IndexError): - log.error('Received invalid message: %s', msg) - return - - if s_estate is None: # drop - return - - log.debug( - '**** Road Router rxMsg **** id=%s estate=%s yard=%s\nmsg=%s', - self.opts.value['id'], - self.road_stack.value.local.name, - self.lane_stack.value.local.name, - msg - ) - - if d_estate is not None and d_estate != self.road_stack.value.local.name: - log.error('Road Router Received message for wrong estate: %s', d_estate) - return - - if d_yard is not None: - # Meant for another yard, send it off! - if d_yard in self.lane_stack.value.nameRemotes: - self.lane_stack.value.transmit(msg, - self.lane_stack.value.nameRemotes[d_yard].uid) - return - return - if d_share is None: - # No queue destination! - log.error('Received message without share: %s', msg) - return - - elif d_share == 'fun': - if self.road_stack.value.kind == kinds.applKinds.minion: - self.fun.value.append(msg) - elif d_share == 'stats_req': - self.stats_req.value.append(msg) - #log.debug("\n**** Stats Request \n %s\n", msg) - - def _process_lane_rxmsg(self, msg, sender): - ''' - Send uxd messages tot he right queue or forward them to the correct - yard etc. - - msg is message body dict - sender is unique name of remote that sent the message - ''' - try: - s_estate, s_yard, s_share = msg['route']['src'] - d_estate, d_yard, d_share = msg['route']['dst'] - except (ValueError, IndexError): - log.error('Lane Router Received invalid message: %s', msg) - return - - if s_yard is None: - return # drop message - - if s_estate is None: # substitute local estate - s_estate = self.road_stack.value.local.name - msg['route']['src'] = (s_estate, s_yard, s_share) - - log.debug( - '**** Lane Router rxMsg **** id=%s estate=%s yard=%s\nmsg=%s', - self.opts.value['id'], - self.road_stack.value.local.name, - self.lane_stack.value.local.name, - msg - ) - - if d_estate is None: - pass - elif d_estate != self.road_stack.value.local.name: - # Forward to the correct estate - if d_estate in self.road_stack.value.nameRemotes: - self.road_stack.value.message(msg, - self.road_stack.value.nameRemotes[d_estate].uid) - return - - if d_yard is None: - pass - elif d_yard != self.lane_stack.value.local.name: - # Meant for another yard, send it off! - if d_yard in self.lane_stack.value.nameRemotes: - self.lane_stack.value.transmit(msg, - self.lane_stack.value.nameRemotes[d_yard].uid) - return - return - if d_share is None: - # No queue destination! - log.error('Lane Router Received message without share: %s', msg) - return - - elif d_share == 'event_req': - self.event_req.value.append(msg) - #log.debug("\n**** Event Subscribe \n %s\n", msg) - elif d_share == 'event_fire': - self.event.value.append(msg) - #log.debug("\n**** Event Fire \n %s\n", msg) - - elif d_share == 'remote_cmd': # assume minion to master or salt-call - if not self.road_stack.value.remotes: - log.error("**** Lane Router: Missing joined master. Unable to route " - "remote_cmd. Requeuing") - self.laters.value.append((msg, sender)) - return - d_estate = self._get_master_estate_name(clustered=self.opts.get('cluster_mode', False)) - if not d_estate: - log.error("**** Lane Router: No available destination estate for 'remote_cmd'." - "Unable to route. Requeuing") - self.laters.value.append((msg, sender)) - return - msg['route']['dst'] = (d_estate, d_yard, d_share) - log.debug("**** Lane Router: Missing destination estate for 'remote_cmd'. " - "Using default route=%s.", msg['route']['dst']) - self.road_stack.value.message(msg, - self.road_stack.value.nameRemotes[d_estate].uid) - - def _get_master_estate_name(self, clustered=False): - ''' - Assign and return the name of the estate for the default master or empty if none - If the default master is no longer available then selects one of the available - masters - ''' - opts = self.opts.value - master = self.road_stack.value.nameRemotes.get(self.master_estate_name.value) - if not master or not master.alived: # select a different master - available_masters = [remote for remote in - list(self.road_stack.value.remotes.values()) - if remote.alived] - if available_masters: - random_master = opts.get('random_master') - if random_master: - master = available_masters[random.randint(0, len(available_masters) - 1)] - else: - master = available_masters[0] - else: - master = None - - self.master_estate_name.value = master.name if master else '' - - return self.master_estate_name.value - - def _availablize(self, minions): - ''' - Return set that is intersection of associated minion estates for - roles in minions and the set of available minion estates. - ''' - suffix = '_{0}'.format(kinds.APPL_KIND_NAMES[kinds.applKinds.minion]) - return list(set(minions) & - set((name.rstrip(suffix) for name in self.availables.value))) - - -class SaltRaetEventer(ioflo.base.deeding.Deed): - ''' - Fire events! - FloScript: - - do salt raet eventer - - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'event_yards': salt.utils.stringutils.to_str('.salt.event.yards'), - 'event': salt.utils.stringutils.to_str('.salt.event.events'), - 'event_req': salt.utils.stringutils.to_str('.salt.event.event_req'), - 'module_refresh': salt.utils.stringutils.to_str('.salt.var.module_refresh'), - 'pillar_refresh': salt.utils.stringutils.to_str('.salt.var.pillar_refresh'), - 'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'), - 'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'), - 'availables': salt.utils.stringutils.to_str('.salt.var.presence.availables'), } - - def _register_event_yard(self, msg): - ''' - register an incoming event request with the requesting yard id - ''' - self.event_yards.value.add(msg['route']['src'][1]) - - def _forward_event(self, msg): - ''' - Forward an event message to all subscribed yards - Event message has a route - ''' - rm_ = [] - if msg.get('tag') == 'pillar_refresh': - self.pillar_refresh.value = True - if msg.get('tag') == 'module_refresh': - self.module_refresh.value = True - for y_name in self.event_yards.value: - if y_name not in self.lane_stack.value.nameRemotes: # subscriber not a remote - rm_.append(y_name) - continue # drop msg don't publish - self.lane_stack.value.transmit(msg, - self.lane_stack.value.fetchUidByName(y_name)) - self.lane_stack.value.serviceAll() - for y_name in rm_: # remove missing subscribers - self.event_yards.value.remove(y_name) - - def action(self): - ''' - Register event requests - Iterate over the registered event yards and fire! - ''' - while self.event_req.value: # event subscription requests are msg with routes - self._register_event_yard( - self.event_req.value.popleft() - ) - - while self.event.value: # events are msgs with routes - self._forward_event( - self.event.value.popleft() - ) - - -class SaltRaetEventerMaster(SaltRaetEventer): - ''' - Fire events! - FloScript: - - do salt raet eventer master - - ''' - def _forward_event(self, msg): - ''' - Forward an event message to all subscribed yards - Event message has a route - Also rebroadcast to all masters in cluster - ''' - super(SaltRaetEventerMaster, self)._forward_event(msg) - if self.opts.value.get('cluster_mode'): - if msg.get('origin') is None: - masters = (self.availables.value & - set((remote.name for remote in list(self.road_stack.value.remotes.values()) - if remote.kind == kinds.applKinds.master))) - for name in masters: - remote = self.road_stack.value.nameRemotes[name] - msg['origin'] = self.road_stack.value.name - s_estate, s_yard, s_share = msg['route']['src'] - msg['route']['src'] = (self.road_stack.value.name, s_yard, s_share) - msg['route']['dst'] = (remote.name, None, 'event_fire') - self.road_stack.value.message(msg, remote.uid) - - -class SaltRaetPresenter(ioflo.base.deeding.Deed): - ''' - Fire presence events! - FloScript: - - do salt raet presenter - - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), - 'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'), - 'alloweds': salt.utils.stringutils.to_str('.salt.var.presence.alloweds'), # odict - 'aliveds': salt.utils.stringutils.to_str('.salt.var.presence.aliveds'), # odict - 'reapeds': salt.utils.stringutils.to_str('.salt.var.presence.reapeds'), # odict - 'availables': salt.utils.stringutils.to_str('.salt.var.presence.availables'), # set - } - - def _send_presence(self, msg): - ''' - Forward an presence message to all subscribed yards - Presence message has a route - ''' - y_name = msg['route']['src'][1] - if y_name not in self.lane_stack.value.nameRemotes: # subscriber not a remote - pass # drop msg don't answer - else: - if 'data' in msg and 'state' in msg['data']: - state = msg['data']['state'] - else: - state = None - - # create answer message - if state in [None, 'available', 'present']: - present = odict() - for name in self.availables.value: - minion = self.aliveds.value.get(name, None) - present[name] = minion.ha[0] if minion else None - data = {'present': present} - else: - # TODO: update to really return joineds - states = {'joined': self.alloweds, - 'allowed': self.alloweds, - 'alived': self.aliveds, - 'reaped': self.reapeds} - try: - minions = states[state].value - except KeyError: - # error: wrong/unknown state requested - log.error('Lane Router Received invalid message: %s', msg) - return - - result = odict() - for name in minions: - result[name] = minions[name].ha[0] - data = {state: result} - - tag = tagify('present', 'presence') - route = {'dst': (None, None, 'event_fire'), - 'src': (None, self.lane_stack.value.local.name, None)} - msg = {'route': route, 'tag': tag, 'data': data} - self.lane_stack.value.transmit(msg, - self.lane_stack.value.fetchUidByName(y_name)) - self.lane_stack.value.serviceAll() - - def action(self): - ''' - Register presence requests - Iterate over the registered presence yards and fire! - ''' - while self.presence_req.value: # presence are msgs with routes - self._send_presence( - self.presence_req.value.popleft() - ) - - -class SaltRaetStatsEventer(ioflo.base.deeding.Deed): - ''' - Fire stats events - FloScript: - - do salt raet state eventer - - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'stats_req': salt.utils.stringutils.to_str('.salt.stats.event_req'), - 'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'), - 'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'), - } - - def _send_stats(self, msg): - ''' - Forward a stats message to all subscribed yards - Stats message has a route - ''' - pass - - def _get_stats(self, tag): - if tag == tagify('road', 'stats'): - return self.road_stack.value.stats - elif tag == tagify('lane', 'stats'): - return self.lane_stack.value.stats - else: - log.error('Missing or invalid tag: %s', tag) - return None - - def action(self): - ''' - Iterate over the registered stats requests and fire! - ''' - while self.stats_req.value: # stats are msgs with routes - self._send_stats( - self.stats_req.value.popleft() - ) - - -class SaltRaetStatsEventerMaster(SaltRaetStatsEventer): - - def _send_stats(self, msg): - ''' - Forward a stats message to all subscribed yards - Stats message has a route - ''' - y_name = msg['route']['src'][1] - if y_name not in self.lane_stack.value.nameRemotes: # subscriber not a remote - return # drop msg don't answer - - stats = self._get_stats(msg.get('tag')) - if stats is None: - return - - route = {'dst': (None, None, 'event_fire'), - 'src': (None, self.lane_stack.value.local.name, None)} - repl = {'route': route, 'tag': msg.get('tag'), 'data': stats} - self.lane_stack.value.transmit(repl, - self.lane_stack.value.fetchUidByName(y_name)) - self.lane_stack.value.serviceAll() - - -class SaltRaetStatsEventerMinion(SaltRaetStatsEventer): - - def _send_stats(self, msg): - ''' - Forward a stats message to all subscribed yards - Stats message has a route - ''' - s_estate, s_yard, s_share = msg['route']['src'] - if s_estate not in self.road_stack.value.nameRemotes: # subscriber not a remote - return # drop msg don't answer - - stats = self._get_stats(msg.get('tag')) - if stats is None: - return - - route = {'dst': (s_estate, s_yard, 'event_fire'), - 'src': (self.road_stack.value.name, self.lane_stack.value.name, None)} - repl = {'route': route, 'tag': msg.get('tag'), 'data': stats} - self.road_stack.value.transmit(repl, - self.road_stack.value.fetchUidByName(s_estate)) - self.road_stack.value.serviceAll() - - -class SaltRaetPublisher(ioflo.base.deeding.Deed): - ''' - Publish to the minions - FloScript: - - do salt raet publisher - - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'publish': salt.utils.stringutils.to_str('.salt.var.publish'), - 'stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'), - 'availables': salt.utils.stringutils.to_str('.salt.var.presence.availables'), - } - - def _publish(self, pub_msg): - ''' - Publish the message out to the targeted minions - ''' - stack = self.stack.value - pub_data = pub_msg['return'] - # only publish to available minions by intersecting sets - - minions = (self.availables.value & - set((remote.name for remote in list(stack.remotes.values()) - if remote.kind in [kinds.applKinds.minion, - kinds.applKinds.syndic]))) - for minion in minions: - uid = self.stack.value.fetchUidByName(minion) - if uid: - route = { - 'dst': (minion, None, 'fun'), - 'src': (self.stack.value.local.name, None, None)} - msg = {'route': route, 'pub': pub_data['pub']} - self.stack.value.message(msg, uid) - - def action(self): - ''' - Pop the publish queue and publish the requests! - ''' - while self.publish.value: - self._publish( - self.publish.value.popleft() - ) - - -class SaltRaetSetupEngines(ioflo.base.deeding.Deed): - ''' - Start the engines! - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'proc_mgr': salt.utils.stringutils.to_str('.salt.usr.proc_mgr')} - - def action(self): - ''' - Only call once, this will start the engine processes - ''' - salt.engines.start_engines(self.opts.value, self.proc_mgr.value) - - -class SaltRaetSetupBeacon(ioflo.base.deeding.Deed): - ''' - Create the Beacon subsystem - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'beacon': salt.utils.stringutils.to_str('.salt.beacon'), - 'modules': salt.utils.stringutils.to_str('.salt.loader.modules')} - - def action(self): - ''' - Run the beacons - ''' - self.beacon.value = salt.beacons.Beacon( - self.opts.value, - self.modules.value) - - -class SaltRaetBeacon(ioflo.base.deeding.Deed): - ''' - Run the beacons - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'modules': salt.utils.stringutils.to_str('.salt.loader.modules'), - 'master_events': salt.utils.stringutils.to_str('.salt.var.master_events'), - 'event': salt.utils.stringutils.to_str('.salt.event.events'), - 'beacon': salt.utils.stringutils.to_str('.salt.beacon')} - - def action(self): - ''' - Run the beacons - ''' - if 'config.merge' in self.modules.value: - b_conf = self.modules.value['config.merge']('beacons') - if b_conf: - try: - events = self.beacon.value.process(b_conf) - self.master_events.value.extend(events) - self.event.value.extend(events) - except Exception: - log.error('Error in the beacon system: ', exc_info=True) - return [] - - -class SaltRaetMasterEvents(ioflo.base.deeding.Deed): - ''' - Take the events off the master event que and send them to the master to - be fired - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'), - 'master_events': salt.utils.stringutils.to_str('.salt.var.master_events')} - - def _prepare(self): - self.master_events.value = deque() - - def action(self): - if not self.master_events.value: - return - events = [] - for master in self.road_stack.value.remotes: - master_uid = master - while self.master_events.value: - events.append(self.master_events.value.popleft()) - route = {'src': (self.road_stack.value.local.name, None, None), - 'dst': (next(list(self.road_stack.value.remotes.values())).name, None, 'remote_cmd')} - load = {'id': self.opts.value['id'], - 'events': events, - 'cmd': '_minion_event'} - self.road_stack.value.transmit({'route': route, 'load': load}, - uid=master_uid) - - -class SaltRaetSetupMatcher(ioflo.base.deeding.Deed): - ''' - Make the matcher object - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'modules': salt.utils.stringutils.to_str('.salt.loader.modules'), - 'matcher': salt.utils.stringutils.to_str('.salt.matcher')} - - def action(self): - self.matcher.value = salt.minion.Matcher( - self.opts.value, - self.modules.value) diff --git a/salt/daemons/flo/dummy.py b/salt/daemons/flo/dummy.py deleted file mode 100644 index 6783da0a5b6..00000000000 --- a/salt/daemons/flo/dummy.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -''' -The dummy publisher for the Salt Master - -Contains functionality to short-circuit a salt-master's -publish functionality so that instead of publications being -sent across the wire, they are instead transparently redirected -to a returner. - -Designed for use primarily in load-testing the salt-master -without the need for a swarm of real minions. -''' - -# pylint: disable=W0232 -# pylint: disable=3rd-party-module-not-gated - -# Import python libs -from __future__ import absolute_import, print_function, unicode_literals -import logging - -# Import salt libs -import ioflo.base.deeding -import salt.utils.stringutils - -log = logging.getLogger(__name__) - - -class SaltDummyPublisher(ioflo.base.deeding.Deed): - ''' - A dummy publisher that transparently redirects publications to - a translation system to have them mocked up and sent back into a router - ''' - Ioinits = { - 'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'publish': salt.utils.stringutils.to_str('.salt.var.publish'), - 'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'), - 'workers': salt.utils.stringutils.to_str('.salt.track.workers'), - } - - def action(self): - while self.publish.value: - pub = self.publish.value.popleft() - log.debug('Dummy publisher publishing: %s', pub) - msg = self._fill_tmpl(pub) - self.lane_stack.value.transmit(msg, self.lane_stack.value.fetchUidByName(next(self.workers.value))) - - def _fill_tmpl(self, pub): - ''' - Takes a template and a job and fills the template with - fake return data associated with the job - ''' - msg = {'load': { - 'fun_args': [], - 'jid': pub['return']['pub']['jid'], - 'return': True, - 'retcode': 0, - 'success': True, - 'cmd': '_return', - 'fun': 'test.ping', - 'id': 'silver' - }, - 'route': { - 'src': ('silver_minion', 'jobber50e73ccefd052167c7', 'jid_ret'), - 'dst': ('silver_master_master', None, 'remote_cmd') - } - } - - log.debug('Dummy publisher faking return with: %s', msg) - return msg diff --git a/salt/daemons/flo/jobber.py b/salt/daemons/flo/jobber.py deleted file mode 100644 index b180213880c..00000000000 --- a/salt/daemons/flo/jobber.py +++ /dev/null @@ -1,398 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Jobber Behaviors -''' -# pylint: disable=W0232 -# pylint: disable=3rd-party-module-not-gated - -# Import python libs -from __future__ import absolute_import, print_function, unicode_literals -import os -import sys -import types -import logging -import traceback -import multiprocessing -import subprocess - -# Import salt libs -from salt.ext import six -import salt.daemons.masterapi -import salt.utils.args -import salt.utils.data -import salt.utils.files -import salt.utils.json -import salt.utils.kinds as kinds -import salt.utils.process -import salt.utils.stringutils -import salt.transport -from raet import raeting, nacling -from raet.lane.stacking import LaneStack -from raet.lane.yarding import RemoteYard - -from salt.utils.platform import is_windows -from salt.utils.event import tagify - -from salt.exceptions import ( - CommandExecutionError, CommandNotFoundError, SaltInvocationError) - -# Import ioflo libs -import ioflo.base.deeding - -from ioflo.base.consoling import getConsole -console = getConsole() -log = logging.getLogger(__name__) - - -@ioflo.base.deeding.deedify( - salt.utils.stringutils.to_str('SaltRaetShellJobberCheck'), - ioinits={'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'grains': salt.utils.stringutils.to_str('.salt.grains'), - 'fun': salt.utils.stringutils.to_str('.salt.var.fun'), - 'matcher': salt.utils.stringutils.to_str('.salt.matcher'), - 'shells': salt.utils.stringutils.to_str('.salt.var.shells'), - 'stack': salt.utils.stringutils.to_str('.salt.road.manor.stack')}) -def jobber_check(self): - ''' - Iterate over the shell jobbers and return the ones that have finished - ''' - rms = [] - for jid in self.shells.value: - if isinstance(self.shells.value[jid]['proc'].poll(), int): - rms.append(jid) - data = self.shells.value[jid] - stdout, stderr = data['proc'].communicate() - ret = salt.utils.json.loads( - stdout, - object_hook=salt.utils.data.encode_dict if six.PY2 else None - )['local'] - route = {'src': (self.stack.value.local.name, 'manor', 'jid_ret'), - 'dst': (data['msg']['route']['src'][0], None, 'remote_cmd')} - ret['cmd'] = '_return' - ret['id'] = self.opts.value['id'] - ret['jid'] = jid - msg = {'route': route, 'load': ret} - master = self.stack.value.nameRemotes.get(data['msg']['route']['src'][0]) - self.stack.value.message( - msg, - master.uid) - for rm_ in rms: - self.shells.value.pop(rm_) - - -@ioflo.base.deeding.deedify( - salt.utils.stringutils.to_str('SaltRaetShellJobber'), - ioinits={'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'grains': salt.utils.stringutils.to_str('.salt.grains'), - 'fun': salt.utils.stringutils.to_str('.salt.var.fun'), - 'matcher': salt.utils.stringutils.to_str('.salt.matcher'), - 'modules': salt.utils.stringutils.to_str('.salt.loader.modules'), - 'shells': {'ipath': salt.utils.stringutils.to_str('.salt.var.shells'), - 'ival': {}}}) -def shell_jobber(self): - ''' - Shell jobber start! - ''' - while self.fun.value: - msg = self.fun.value.popleft() - data = msg.get('pub') - match = getattr( - self.matcher.value, - '{0}_match'.format( - data.get('tgt_type', 'glob') - ) - )(data['tgt']) - if not match: - continue - fun = data['fun'] - if fun in self.modules.value: - func = self.modules.value[fun] - else: - continue - args, kwargs = salt.minion.load_args_and_kwargs( - func, - salt.utils.args.parse_input( - data['arg'], - no_parse=data.get('no_parse', [])), - data) - cmd = ['salt-call', - '--out', 'json', - '--metadata', - '-c', salt.syspaths.CONFIG_DIR] - if 'return' in data: - cmd.append('--return') - cmd.append(data['return']) - cmd.append(fun) - for arg in args: - cmd.append(arg) - for key in kwargs: - cmd.append('{0}={1}'.format(key, kwargs[key])) - que = {'pub': data, - 'msg': msg} - que['proc'] = subprocess.Popen( - cmd, - shell=False, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE) - self.shells.value[data['jid']] = que - - -class SaltRaetNixJobber(ioflo.base.deeding.Deed): - ''' - Execute a function call job on a minion on a *nix based system - FloScript: - - do salt raet nix jobber - - ''' - Ioinits = {'opts_store': salt.utils.stringutils.to_str('.salt.opts'), - 'grains': salt.utils.stringutils.to_str('.salt.grains'), - 'modules': salt.utils.stringutils.to_str('.salt.loader.modules'), - 'returners': salt.utils.stringutils.to_str('.salt.loader.returners'), - 'module_executors': salt.utils.stringutils.to_str('.salt.loader.executors'), - 'fun': salt.utils.stringutils.to_str('.salt.var.fun'), - 'matcher': salt.utils.stringutils.to_str('.salt.matcher'), - 'executors': salt.utils.stringutils.to_str('.salt.track.executors'), - 'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'), } - - def _prepare(self): - ''' - Map opts for convenience - ''' - self.opts = self.opts_store.value - self.proc_dir = salt.minion.get_proc_dir(self.opts['cachedir']) - self.serial = salt.payload.Serial(self.opts) - self.executors.value = {} - - def _setup_jobber_stack(self): - ''' - Setup and return the LaneStack and Yard used by the jobber yard - to communicate with the minion manor yard - - ''' - role = self.opts.get('id', '') - if not role: - emsg = ("Missing role required to setup Jobber Lane.") - log.error(emsg + "\n") - raise ValueError(emsg) - - kind = self.opts['__role'] - if kind not in kinds.APPL_KINDS: - emsg = ("Invalid application kind = '{0}' for Jobber lane.".format(kind)) - log.error(emsg + "\n") - raise ValueError(emsg) - - if kind == 'minion': - lanename = "{0}_{1}".format(role, kind) - else: - emsg = ("Unsupported application kind = '{0}' for Jobber Lane.".format(kind)) - log.error(emsg + '\n') - raise ValueError(emsg) - - sockdirpath = self.opts['sock_dir'] - name = 'jobber' + nacling.uuid(size=18) - stack = LaneStack( - name=name, - lanename=lanename, - sockdirpath=sockdirpath) - - stack.Pk = raeting.PackKind.pack.value - # add remote for the manor yard - stack.addRemote(RemoteYard(stack=stack, - name='manor', - lanename=lanename, - dirpath=sockdirpath)) - console.concise("Created Jobber Stack {0}\n".format(stack.name)) - return stack - - def _return_pub(self, msg, ret, stack): - ''' - Send the return data back via the uxd socket - ''' - route = {'src': (self.road_stack.value.local.name, stack.local.name, 'jid_ret'), - 'dst': (msg['route']['src'][0], None, 'remote_cmd')} - mid = self.opts['id'] - ret['cmd'] = '_return' - ret['id'] = mid - try: - oput = self.modules.value[ret['fun']].__outputter__ - except (KeyError, AttributeError, TypeError): - pass - else: - if isinstance(oput, six.string_types): - ret['out'] = oput - msg = {'route': route, 'load': ret} - stack.transmit(msg, stack.fetchUidByName('manor')) - stack.serviceAll() - - def action(self): - ''' - Pull the queue for functions to execute - ''' - while self.fun.value: - msg = self.fun.value.popleft() - data = msg.get('pub') - match = getattr( - self.matcher.value, - '{0}_match'.format( - data.get('tgt_type', 'glob') - ) - )(data['tgt']) - if not match: - continue - if 'user' in data: - log.info( - 'User %s Executing command %s with jid %s', - data['user'], data['fun'], data['jid'] - ) - else: - log.info( - 'Executing command %s with jid %s', - data['fun'], data['jid'] - ) - log.debug('Command details %s', data) - - if is_windows(): - # SaltRaetNixJobber is not picklable. Pickling is necessary - # when spawning a process in Windows. Since the process will - # be spawned and joined on non-Windows platforms, instead of - # this, just run the function directly and absorb any thrown - # exceptions. - try: - self.proc_run(msg) - except Exception as exc: - log.error('Exception caught by jobber: %s', exc, exc_info=True) - else: - process = multiprocessing.Process( - target=self.proc_run, - kwargs={'msg': msg} - ) - process.start() - process.join() - - def proc_run(self, msg): - ''' - Execute the run in a dedicated process - ''' - data = msg['pub'] - fn_ = os.path.join(self.proc_dir, data['jid']) - self.opts['__ex_id'] = data['jid'] - salt.utils.process.daemonize_if(self.opts) - - salt.transport.jobber_stack = stack = self._setup_jobber_stack() - # set up return destination from source - src_estate, src_yard, src_share = msg['route']['src'] - salt.transport.jobber_estate_name = src_estate - salt.transport.jobber_yard_name = src_yard - - sdata = {'pid': os.getpid()} - sdata.update(data) - with salt.utils.files.fopen(fn_, 'w+b') as fp_: - fp_.write(self.serial.dumps(sdata)) - ret = {'success': False} - function_name = data['fun'] - if function_name in self.modules.value: - try: - func = self.modules.value[data['fun']] - args, kwargs = salt.minion.load_args_and_kwargs( - func, - salt.utils.args.parse_input( - data['arg'], - no_parse=data.get('no_parse', [])), - data) - sys.modules[func.__module__].__context__['retcode'] = 0 - - executors = data.get('module_executors') or self.opts.get('module_executors', ['direct_call']) - if isinstance(executors, six.string_types): - executors = [executors] - elif not isinstance(executors, list) or not executors: - raise SaltInvocationError( - 'Wrong executors specification: {0}. String or ' - 'non-empty list expected'.format(executors) - ) - if self.opts.get('sudo_user', '') and executors[-1] != 'sudo': - executors[-1] = 'sudo' # replace - log.trace("Executors list %s", executors) - - for name in executors: - fname = '{0}.execute'.format(name) - if fname not in self.module_executors.value: - raise SaltInvocationError("Executor '{0}' is not available".format(name)) - return_data = self.module_executors.value[fname](self.opts, data, func, args, kwargs) - if return_data is not None: - break - - if isinstance(return_data, types.GeneratorType): - ind = 0 - iret = {} - for single in return_data: - if isinstance(single, dict) and isinstance(iret, list): - iret.update(single) - else: - if not iret: - iret = [] - iret.append(single) - tag = tagify( - [data['jid'], 'prog', self.opts['id'], six.text_type(ind)], - 'job') - event_data = {'return': single} - self._fire_master(event_data, tag) # Need to look into this - ind += 1 - ret['return'] = iret - else: - ret['return'] = return_data - ret['retcode'] = sys.modules[func.__module__].__context__.get( - 'retcode', - 0 - ) - ret['success'] = True - except CommandNotFoundError as exc: - msg = 'Command required for \'{0}\' not found'.format( - function_name - ) - log.debug(msg, exc_info=True) - ret['return'] = '{0}: {1}'.format(msg, exc) - except CommandExecutionError as exc: - log.error( - 'A command in \'%s\' had a problem: %s', - function_name, exc, - exc_info_on_loglevel=logging.DEBUG - ) - ret['return'] = 'ERROR: {0}'.format(exc) - except SaltInvocationError as exc: - log.error( - 'Problem executing \'%s\': %s', - function_name, exc, - exc_info_on_loglevel=logging.DEBUG - ) - ret['return'] = 'ERROR executing \'{0}\': {1}'.format( - function_name, exc - ) - except TypeError as exc: - msg = ('TypeError encountered executing {0}: {1}. See ' - 'debug log for more info.').format(function_name, exc) - log.warning(msg, exc_info_on_loglevel=logging.DEBUG) - ret['return'] = msg - except Exception: - msg = 'The minion function caused an exception' - log.warning(msg, exc_info_on_loglevel=logging.DEBUG) - ret['return'] = '{0}: {1}'.format(msg, traceback.format_exc()) - else: - ret['return'] = '\'{0}\' is not available.'.format(function_name) - - ret['jid'] = data['jid'] - ret['fun'] = data['fun'] - ret['fun_args'] = data['arg'] - self._return_pub(msg, ret, stack) - if data['ret']: - ret['id'] = self.opts['id'] - for returner in set(data['ret'].split(',')): - try: - self.returners.value['{0}.returner'.format( - returner - )](ret) - except Exception as exc: - log.error('The return failed for job %s %s', data['jid'], exc) - console.concise("Closing Jobber Stack {0}\n".format(stack.name)) - stack.server.close() - salt.transport.jobber_stack = None diff --git a/salt/daemons/flo/maint.flo b/salt/daemons/flo/maint.flo deleted file mode 100644 index f6ae2b7e90b..00000000000 --- a/salt/daemons/flo/maint.flo +++ /dev/null @@ -1,28 +0,0 @@ -# Maintenance process floscript - -house master - -framer jobcleaner be active first setup - frame setup - print Setup Maint - enter - do salt raet maint setup - go modules - frame modules - enter - do salt load modules - go fsclean - frame fsclean - enter - do salt raet maint fileserver clean - go start - frame start - do salt raet maint old jobs clear - -framer backends be active first start - frame start - do salt raet maint backends update - -framer scheduler be active first start - frame start - do salt schedule diff --git a/salt/daemons/flo/maint.py b/salt/daemons/flo/maint.py deleted file mode 100644 index 98a754f321a..00000000000 --- a/salt/daemons/flo/maint.py +++ /dev/null @@ -1,152 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Define the behaviors used in the maintenance process -''' -# pylint: disable=3rd-party-module-not-gated -from __future__ import absolute_import, print_function, unicode_literals -# Import python libs -import multiprocessing -import os - -# Import ioflo libs -import ioflo.base.deeding - -# Import salt libs -import salt.daemons.masterapi -import salt.fileserver -import salt.loader -import salt.utils.minions -import salt.utils.stringutils - - -@ioflo.base.deeding.deedify( - salt.utils.stringutils.to_str('SaltRaetMaintFork'), - ioinits={'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'proc_mgr': salt.utils.stringutils.to_str('.salt.usr.proc_mgr')}) -def maint_fork(self): - ''' - For off the maintinence process from the master router process - FloScript: - - do salt raet maint fork at enter - ''' - self.proc_mgr.value.add_process(Maintenance, args=(self.opts.value,)) - - -class Maintenance(multiprocessing.Process): - ''' - Start the maintinance process within ioflo - ''' - def __init__(self, opts): - super(Maintenance, self).__init__() - self.opts = opts - - def run(self): - ''' - Spin up a worker, do this in s multiprocess - ''' - behaviors = ['salt.daemons.flo'] - preloads = [(salt.utils.stringutils.to_str('.salt.opts'), dict(value=self.opts))] - - console_logdir = self.opts.get('ioflo_console_logdir', '') - if console_logdir: - consolepath = os.path.join(console_logdir, 'maintenance.log') - else: # empty means log to std out - consolepath = '' - - ioflo.app.run.start( - name='maintenance', - period=float(self.opts['loop_interval']), - stamp=0.0, - real=self.opts['ioflo_realtime'], - filepath=self.opts['maintenance_floscript'], - behaviors=behaviors, - username="", - password="", - mode=None, - houses=None, - metas=None, - preloads=preloads, - verbose=int(self.opts['ioflo_verbose']), - consolepath=consolepath, - ) - - -class SaltRaetMaintSetup(ioflo.base.deeding.Deed): - ''' - Init loader objects used - FloScript: - - do salt raet maint setup at enter - - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'fileserver': salt.utils.stringutils.to_str('.salt.loader.fileserver'), - 'runners': salt.utils.stringutils.to_str('.salt.loader.runners'), - 'pillargitfs': salt.utils.stringutils.to_str('.salt.loader.pillargitfs'), - 'ckminions': salt.utils.stringutils.to_str('.salt.loader.ckminions')} - - def action(self): - ''' - Set up the objects used in the maint process - ''' - self.fileserver.value = salt.fileserver.Fileserver(self.opts.value) - self.runners.value = salt.loader.runner(self.opts.value) - self.ckminions.value = salt.utils.minions.CkMinions(self.opts.value) - self.pillargitfs.value = salt.daemons.masterapi.init_git_pillar( - self.opts.value) - - -class SaltRaetMaintFileserverClean(ioflo.base.deeding.Deed): - ''' - Clear the fileserver backend caches - FloScript: - - do salt raet maint fileserver clean at enter - - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts')} - - def action(self): - ''' - Clean! - ''' - salt.daemons.masterapi.clean_fsbackend(self.opts.value) - - -class SaltRaetMaintOldJobsClear(ioflo.base.deeding.Deed): - ''' - Iterate over the jobs directory and clean out the old jobs - FloScript: - - do salt raet maint old jobs clear - - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts')} - - def action(self): - ''' - Clear out the old jobs cache - ''' - salt.daemons.masterapi.clean_old_jobs(self.opts.value) - - -class SaltRaetMaintBackendsUpdate(ioflo.base.deeding.Deed): - ''' - Update the fileserver and external pillar caches - FloScript: - - do salt raet maint backends update - - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'fileserver': salt.utils.stringutils.to_str('.salt.loader.fileserver'), - 'pillargitfs': salt.utils.stringutils.to_str('.salt.loader.pillargitfs')} - - def action(self): - ''' - Update! - ''' - for pillargit in self.pillargitfs.value: - pillargit.update() - salt.daemons.masterapi.fileserver_update(self.fileserver.value) diff --git a/salt/daemons/flo/master.flo b/salt/daemons/flo/master.flo deleted file mode 100644 index 721a5a4a43b..00000000000 --- a/salt/daemons/flo/master.flo +++ /dev/null @@ -1,202 +0,0 @@ -# Salt Master Floscript - -house master - -framer masterudpstack be active first setup - # Begin the pre-flight checks - frame setup - enter - do salt raet cleanup - do salt raet road stack setup per inode ".salt.road.manor" - do salt raet manor lane setup per inode ".salt.lane.manor" - do salt raet process manager setup - go zmqstart - go spawnmaint - - frame zmqstart - # Start the zmq ret port if configured - let me if .salt.etc.zmq_behavior - do salt zmq setup at enter - bid start zmqret - go spawnmaint - - # Create the maintanence frame - frame spawnmaint - enter - do salt raet maint fork - go spawnworkers - - # Start forking master workers - frame spawnworkers - enter - do salt raet worker fork - go spawnreactor - go spawneventreturn - go startengines - go start - - frame spawnreactor - let me if .salt.etc.reactor - enter - do salt raet reactor fork - go spawneventreturn - go startengines - go start - - frame spawneventreturn - let me if .salt.etc.event_return - enter - do salt raet event return fork - go startengines - go start - - frame startengines - let me if .salt.etc.engines - enter - do salt raet setup engines - go start - - frame start - # Start the message receive framer - bid start inbound - # Start the cluster bootstrap framer - bid start bootstrap - # Start the message receive framer - bid start uxdrouter - # Start the event framer - bid start events - # Start the presence framer - bid start presence - # Start the publish framer - bid start publish - # Start the manage framer - bid start manager - # Start the outbound framer - bid start outbound - exit - do salt raet road stack closer per inode ".salt.road.manor." - do salt raet lane stack closer per inode ".salt.lane.manor." - -######################################### -# Main RAET Behaviors # -######################################### - -# Inbound framer -framer inbound be inactive first start - frame start - do salt raet road stack service rx - do salt raet lane stack service rx - -# Bootstrap framer -framer bootstrap be inactive first setup - frame setup - enter - do salt raet road clustered per inode ".salt.road.manor." - - go clustermaster - go quit - - frame clustermaster - let if salt.road.manor.cluster.clustered - print Setting Up Master Cluster .... - do salt raet road usher master setup per inode ".salt.road.manor." - go join - - frame join - print Joining... - enter - do salt raet road stack joiner per inode ".salt.road.manor." - recur - do salt raet road stack joined per inode ".salt.road.manor." - do salt raet road stack rejected per inode ".salt.road.manor." - - go next if joined in .salt.road.manor.status - #go abort if rejected in .salt.road.manor.status - - frame joined - print Joined - go next if elapsed >= 0.5 - - frame allow - print Allowing... - enter - do salt raet road stack allower per inode ".salt.road.manor." - recur - do salt raet road stack allowed per inode ".salt.road.manor." - - go next if allowed in .salt.road.manor.status - - frame allowed - print Allowed - go next - - frame clustering - print Cluster Setup ... - do salt raet road cluster load setup - go next - - frame quit - bid stop me - - frame abort - bid stop all - -# Router framer -framer uxdrouter be inactive first start - frame start - do salt raet router master - -# Event bus framer -framer events be inactive first start - frame start - do salt raet eventer master - do salt raet stats eventer master - -# Presence framer -framer presence be inactive first start - frame start - go multiheaded - go raet - - frame raet - do salt raet presenter - - frame multiheaded - let me if .salt.etc.zmq_behavior - do salt zmq crypticle setup at enter - do salt zmq publisher - do salt raet presenter - - - - -# Publisher framer -framer publish be inactive first setup - frame setup - go dummy_publisher - go raet_publisher - frame dummy_publisher - let me if .salt.etc.dummy_pub - do salt dummy publisher - frame raet_publisher - do salt raet publisher - -# Manager framer -framer manager be inactive first start at 10.0 - frame start - do salt raet road stack manager per inode ".salt.road.manor" - -# Outbound framer -framer outbound be inactive first start - frame start - do salt raet lane stack service tx - do salt raet road stack service tx - -######################################### -# Main ZMQ Behaviors # -######################################### - -framer zmqret be inactive first start - frame start - enter - do salt zmq ret fork diff --git a/salt/daemons/flo/minion.flo b/salt/daemons/flo/minion.flo deleted file mode 100644 index 85adfb84098..00000000000 --- a/salt/daemons/flo/minion.flo +++ /dev/null @@ -1,187 +0,0 @@ -# Salt Minion floscript - -house minion - -framer minionudpstack be active first setup - # Begin the pre-flight checks - frame setup - enter - do salt raet cleanup - do salt raet road stack setup per inode ".salt.road.manor" - do salt raet manor lane setup - do salt raet process manager setup - go loadmodules - - # Load the minion mods - frame loadmodules - do salt load modules at enter - go setupmatcher - - frame setupmatcher - do salt raet setup matcher at enter - go setupbeacon - - frame setupbeacon - do salt raet setup beacon at enter - go spawnreactor - go startengines - go start - - frame spawnreactor - let me if .salt.etc.reactor - enter - do salt raet reactor fork - go startengines - go start - - frame startengines - let me if .salt.etc.engines - enter - do salt raet setup engines - go start - - # OK, let's start the minion up - frame start - # Start the inbound framer - bid start inbound - # Start the bootstrap framer - bid start bootstrap - # Start the eventing framer - bid start eventing - # Start the functionmanage framer - bid start functionmanager - # Start the outbound framer - bid start outbound - # Start the scheduler - bid start scheduler - - # Cleanup on exit - exit - do salt raet road stack closer per inode ".salt.road.manor." - do salt raet lane stack closer per inode ".salt.lane.manor." - -# Framer for handling inbound traffic -framer inbound be inactive first start - frame start - do salt raet road stack service rx - do salt raet lane stack service rx - -framer bootstrap be inactive first setup - - frame setup - enter - do salt raet road clustered per inode ".salt.road.manor." - do salt raet road usher minion setup per inode ".salt.road.manor." - go clustermaster - go multimaster - - frame clustermaster - let if salt.road.manor.cluster.clustered - print Setting Up Master Cluster .... - go join - - frame multimaster - print Setting Up Master or MultiMaster - go join - - frame join - print Joining... - enter - do salt raet road stack joiner per inode ".salt.road.manor." - recur - do salt raet road stack joined per inode ".salt.road.manor." - do salt raet road stack rejected per inode ".salt.road.manor." - - go next if joined in .salt.road.manor.status - go abort if rejected in .salt.road.manor.status - - frame joined - print Joined - go next if elapsed >= 0.5 - - frame allow - print Allowing... - enter - do salt raet road stack allower per inode ".salt.road.manor." - recur - do salt raet road stack allowed per inode ".salt.road.manor." - - go next if allowed in .salt.road.manor.status - - frame allowed - print Allowed - go next if elapsed >= 0.5 - - frame clustering - print Cluster Setup ... - do salt raet road cluster load setup - go next - - frame pillar - print Pillaring - enter - do salt load pillar - go loading - - frame loading - print Loading - enter - do salt load modules - go latestart - - frame latestart - # Start late frames that need the pillar/modules to be available - # Start the master events loop - bid start masterevents - # Start Beacon - bid start beacon - go router - - frame router - # start the manager framer - bid start manager #start alive presence from minion side - do salt raet router minion - go pillar if .salt.var.pillar_refresh - go loading if .salt.var.module_refresh - - frame abort - bid stop all - -framer eventing be inactive first event - frame event - do salt raet eventer - do salt raet stats eventer minion - -framer functionmanager be inactive first setup - frame setup - go shell - go nix - frame shell - let me if .salt.etc.shell_jobber - do salt raet shell jobber - do salt raet shell jobber check - frame nix - do salt raet nix jobber - -framer manager be inactive first start at 10.0 - frame start - do salt raet road stack manager per inode ".salt.road.manor" - -framer beacon be inactive first start - frame start - do salt raet beacon - -framer masterevents be inactive first start - frame start - do salt raet master events - -# Framer for handling outbound traffic -framer outbound be inactive first start - frame start - do salt raet lane stack service tx - do salt raet road stack service tx - - -framer scheduler be inactive first start - frame start - do salt schedule diff --git a/salt/daemons/flo/reactor.py b/salt/daemons/flo/reactor.py deleted file mode 100644 index 9337367ac50..00000000000 --- a/salt/daemons/flo/reactor.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Start the reactor! -''' -# pylint: disable=3rd-party-module-not-gated -from __future__ import absolute_import, print_function, unicode_literals -# Import salt libs -import salt.utils.reactor -import salt.utils.event -import salt.utils.stringutils -# Import ioflo libs -import ioflo.base.deeding - - -@ioflo.base.deeding.deedify( - salt.utils.stringutils.to_str('SaltRaetReactorFork'), - ioinits={ - 'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'proc_mgr': salt.utils.stringutils.to_str('.salt.usr.proc_mgr')}) -def reactor_fork(self): - ''' - Add a reactor object to the process manager - ''' - self.proc_mgr.value.add_process( - salt.utils.reactor.Reactor, - args=(self.opts.value,)) - - -@ioflo.base.deeding.deedify( - salt.utils.stringutils.to_str('SaltRaetEventReturnFork'), - ioinits={ - 'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'proc_mgr': salt.utils.stringutils.to_str('.salt.usr.proc_mgr')}) -def event_return_fork(self): - ''' - Add a reactor object to the process manager - ''' - self.proc_mgr.value.add_process( - salt.utils.event.EventReturn, - args=(self.opts.value,)) diff --git a/salt/daemons/flo/worker.flo b/salt/daemons/flo/worker.flo deleted file mode 100644 index e7c139295b2..00000000000 --- a/salt/daemons/flo/worker.flo +++ /dev/null @@ -1,30 +0,0 @@ -# Salt Master Worker Floscript. -# -# This controls a single worker proc, many are started based on the value in worker_threads - -house worker - -framer uxdrouter be active first setup - frame setup - do salt raet worker setup at enter - go zmqsetup - go raetstart - - frame zmqsetup - # Only enter if configured to run dual head - let me if .salt.etc.zmq_behavior - bid start zmqworker - go raetstart - - frame raetstart - bid start raetworker - -framer raetworker be inactive first start - frame start - do salt raet worker router - exit - do salt raet lane stack closer per inode ".salt.lane.manor." - -framer zmqworker be inactive first start - frame start - do salt zmq worker diff --git a/salt/daemons/flo/worker.py b/salt/daemons/flo/worker.py deleted file mode 100644 index 7975a0a1a01..00000000000 --- a/salt/daemons/flo/worker.py +++ /dev/null @@ -1,249 +0,0 @@ -# -*- coding: utf-8 -*- -''' -The core behaviors used by minion and master -''' -# pylint: disable=W0232 -# pylint: disable=3rd-party-module-not-gated - -from __future__ import absolute_import, print_function, unicode_literals - -# Import python libs -import time -import os -import multiprocessing -import logging -from salt.ext.six.moves import range - -# Import salt libs -import salt.daemons.flo -import salt.daemons.masterapi -from raet import raeting -from raet.lane.stacking import LaneStack -from raet.lane.yarding import RemoteYard - -import salt.utils.kinds as kinds -import salt.utils.stringutils - -# Import ioflo libs -import ioflo.base.deeding - -log = logging.getLogger(__name__) - -# convert to set once list is larger than about 3 because set hashes -INHIBIT_RETURN = [] # ['_return'] # cmd for which we should not send return - - -@ioflo.base.deeding.deedify( - salt.utils.stringutils.to_str('SaltRaetWorkerFork'), - ioinits={ - 'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'proc_mgr': salt.utils.stringutils.to_str('.salt.usr.proc_mgr'), - 'worker_verify': salt.utils.stringutils.to_str('.salt.var.worker_verify'), - 'access_keys': salt.utils.stringutils.to_str('.salt.access_keys'), - 'mkey': salt.utils.stringutils.to_str('.salt.var.zmq.master_key'), - 'aes': salt.utils.stringutils.to_str('.salt.var.zmq.aes')}) -def worker_fork(self): - ''' - Fork off the worker procs - FloScript: - - do salt raet worker fork at enter - ''' - for index in range(int(self.opts.value['worker_threads'])): - time.sleep(0.01) - self.proc_mgr.value.add_process( - Worker, - args=( - self.opts.value, - index + 1, - self.worker_verify.value, - self.access_keys.value, - self.mkey.value, - self.aes.value - ) - ) - - -class Worker(multiprocessing.Process): - ''' - Create an ioflo worker in a separate process - ''' - def __init__(self, opts, windex, worker_verify, access_keys, mkey, aes): - super(Worker, self).__init__() - self.opts = opts - self.windex = windex - self.worker_verify = worker_verify - self.access_keys = access_keys - self.mkey = mkey - self.aes = aes - - def run(self): - ''' - Spin up a worker, do this in multiprocess - windex is worker index - ''' - self.opts['__worker'] = True - behaviors = ['salt.daemons.flo'] - preloads = [(salt.utils.stringutils.to_str('.salt.opts'), dict(value=self.opts)), - (salt.utils.stringutils.to_str('.salt.var.worker_verify'), - dict(value=self.worker_verify))] - preloads.append((salt.utils.stringutils.to_str('.salt.var.fork.worker.windex'), - dict(value=self.windex))) - preloads.append((salt.utils.stringutils.to_str('.salt.var.zmq.master_key'), - dict(value=self.mkey))) - preloads.append((salt.utils.stringutils.to_str('.salt.var.zmq.aes'), dict(value=self.aes))) - preloads.append((salt.utils.stringutils.to_str('.salt.access_keys'), - dict(value=self.access_keys))) - preloads.extend(salt.daemons.flo.explode_opts(self.opts)) - - console_logdir = self.opts.get('ioflo_console_logdir', '') - if console_logdir: - consolepath = os.path.join(console_logdir, "worker_{0}.log".format(self.windex)) - else: # empty means log to std out - consolepath = '' - - ioflo.app.run.start( - name='worker{0}'.format(self.windex), - period=float(self.opts['ioflo_period']), - stamp=0.0, - real=self.opts['ioflo_realtime'], - filepath=self.opts['worker_floscript'], - behaviors=behaviors, - username='', - password='', - mode=None, - houses=None, - metas=None, - preloads=preloads, - verbose=int(self.opts['ioflo_verbose']), - consolepath=consolepath, - ) - - -class SaltRaetWorkerSetup(ioflo.base.deeding.Deed): - ''' - FloScript: - - do salt raet worker setup at enter - - ''' - Ioinits = { - 'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'windex': salt.utils.stringutils.to_str('.salt.var.fork.worker.windex'), - 'access_keys': salt.utils.stringutils.to_str('.salt.access_keys'), - 'remote_loader': salt.utils.stringutils.to_str('.salt.loader.remote'), - 'local_loader': salt.utils.stringutils.to_str('.salt.loader.local'), - 'inode': salt.utils.stringutils.to_str('.salt.lane.manor.'), - 'stack': salt.utils.stringutils.to_str('stack'), - 'local': {'ipath': salt.utils.stringutils.to_str('local'), - 'ival': {'lanename': 'master'}} - } - - def action(self): - ''' - Set up the uxd stack and behaviors - ''' - name = "worker{0}".format(self.windex.value) - # master application kind - kind = self.opts.value['__role'] - if kind not in kinds.APPL_KINDS: - emsg = ("Invalid application kind = '{0}' for Master Worker.".format(kind)) - log.error(emsg + "\n") - raise ValueError(emsg) - if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.master], - kinds.APPL_KIND_NAMES[kinds.applKinds.syndic]]: - lanename = 'master' - else: # workers currently are only supported for masters - emsg = ("Invalid application kind '{0}' for Master Worker.".format(kind)) - log.error(emsg + '\n') - raise ValueError(emsg) - sockdirpath = self.opts.value['sock_dir'] - self.stack.value = LaneStack( - name=name, - lanename=lanename, - sockdirpath=sockdirpath) - self.stack.value.Pk = raeting.PackKind.pack.value - manor_yard = RemoteYard( - stack=self.stack.value, - name='manor', - lanename=lanename, - dirpath=sockdirpath) - self.stack.value.addRemote(manor_yard) - self.remote_loader.value = salt.daemons.masterapi.RemoteFuncs( - self.opts.value) - self.local_loader.value = salt.daemons.masterapi.LocalFuncs( - self.opts.value, - self.access_keys.value) - init = {} - init['route'] = { - 'src': (None, self.stack.value.local.name, None), - 'dst': (None, manor_yard.name, 'worker_req') - } - self.stack.value.transmit(init, self.stack.value.fetchUidByName(manor_yard.name)) - self.stack.value.serviceAll() - - def __del__(self): - self.stack.server.close() - - -class SaltRaetWorkerRouter(ioflo.base.deeding.Deed): - ''' - FloScript: - - do salt raet worker router - - ''' - Ioinits = { - 'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'), - 'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'), - 'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'worker_verify': salt.utils.stringutils.to_str('.salt.var.worker_verify'), - 'remote_loader': salt.utils.stringutils.to_str('.salt.loader.remote'), - 'local_loader': salt.utils.stringutils.to_str('.salt.loader.local'), - } - - def action(self): - ''' - Read in a command and execute it, send the return back up to the - main master process - ''' - self.lane_stack.value.serviceAll() - while self.lane_stack.value.rxMsgs: - msg, sender = self.lane_stack.value.rxMsgs.popleft() - try: - s_estate, s_yard, s_share = msg['route']['src'] - d_estate, d_yard, d_share = msg['route']['dst'] - except (ValueError, IndexError): - log.error('Received invalid message: %s', msg) - return - - log.debug("**** Worker Router rxMsg\nmsg=%s", msg) - - if 'load' in msg: - cmd = msg['load'].get('cmd') - if not cmd: - continue - elif cmd.startswith('__'): - continue - ret = {} - if d_share == 'remote_cmd': - if hasattr(self.remote_loader.value, cmd): - ret['return'] = getattr(self.remote_loader.value, cmd)(msg['load']) - elif d_share == 'local_cmd': - if hasattr(self.local_loader.value, cmd): - ret['return'] = getattr(self.local_loader.value, cmd)(msg['load']) - else: - ret = {'error': 'Invalid request'} - if cmd == 'publish' and 'pub' in ret.get('return', {}): - r_share = 'pub_ret' - ret['__worker_verify'] = self.worker_verify.value - else: - r_share = s_share - if cmd not in INHIBIT_RETURN: - ret['route'] = { - 'src': (None, self.lane_stack.value.local.name, None), - 'dst': (s_estate, s_yard, r_share) - } - self.lane_stack.value.transmit(ret, - self.lane_stack.value.fetchUidByName('manor')) - self.lane_stack.value.serviceAll() diff --git a/salt/daemons/flo/zero.py b/salt/daemons/flo/zero.py deleted file mode 100644 index bf4e5725eca..00000000000 --- a/salt/daemons/flo/zero.py +++ /dev/null @@ -1,249 +0,0 @@ -# -*- coding: utf-8 -*- -''' -IoFlo behaviors for running a ZeroMQ based master -''' -# pylint: disable=W0232 -# pylint: disable=3rd-party-module-not-gated - -# Import python libs -from __future__ import absolute_import, print_function, unicode_literals -import os -import logging -import hashlib -import multiprocessing -import errno -# Import ioflo libs -import ioflo.base.deeding -# Import third party libs -from salt.utils.zeromq import zmq -import salt.master -import salt.crypt -import salt.daemons.masterapi -import salt.payload -import salt.utils.stringutils - -log = logging.getLogger(__name__) - - -class SaltZmqSetup(ioflo.base.deeding.Deed): - ''' - do salt zmq setup at enter - - Setup shares - .salt.var.zmq.master_key - .salt.var.zmq.aet share - - This behavior must be run before any other zmq related - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'mkey': salt.utils.stringutils.to_str('.salt.var.zmq.master_key'), - 'aes': salt.utils.stringutils.to_str('.salt.var.zmq.aes')} - - def action(self): - ''' - Assign master key to .salt.var.zmq.master_key - Copy opts['aes'] to .salt.var.zmq.aes - ''' - self.mkey.value = salt.crypt.MasterKeys(self.opts.value) - self.aes.value = self.opts.value['aes'] - - -@ioflo.base.deeding.deedify( - salt.utils.stringutils.to_str('SaltZmqRetFork'), - ioinits={ - 'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'proc_mgr': salt.utils.stringutils.to_str('.salt.usr.proc_mgr'), - 'mkey': salt.utils.stringutils.to_str('.salt.var.zmq.master_key'), - 'aes': salt.utils.stringutils.to_str('.salt.var.zmq.aes')}) -def zmq_ret_fork(self): - ''' - Create the forked process for the ZeroMQ Ret Port - ''' - self.proc_mgr.value.add_process( - ZmqRet, - args=( - self.opts.value, - self.mkey.value, - self.aes.value)) - - -class ZmqRet(multiprocessing.Process): - ''' - Create the forked process for the ZeroMQ Ret Port - ''' - def __init__(self, opts, mkey, aes): - self.opts = opts - self.mkey = mkey - self.aes = aes - super(ZmqRet, self).__init__() - - def run(self): - ''' - Start the ret port binding - ''' - self.context = zmq.Context(self.opts['worker_threads']) - self.uri = 'tcp://{interface}:{ret_port}'.format(**self.opts) - log.info('ZMQ Ret port binding to %s', self.uri) - self.clients = self.context.socket(zmq.ROUTER) - if self.opts['ipv6'] is True and hasattr(zmq, 'IPV4ONLY'): - # IPv6 sockets work for both IPv6 and IPv4 addresses - self.clients.setsockopt(zmq.IPV4ONLY, 0) - try: - self.clients.setsockopt(zmq.HWM, self.opts['rep_hwm']) - except AttributeError: - self.clients.setsockopt(zmq.SNDHWM, self.opts['rep_hwm']) - self.clients.setsockopt(zmq.RCVHWM, self.opts['rep_hwm']) - self.clients.setsockopt(zmq.BACKLOG, self.opts['zmq_backlog']) - self.workers = self.context.socket(zmq.DEALER) - self.w_uri = 'ipc://{0}'.format( - os.path.join(self.opts['sock_dir'], 'workers.ipc') - ) - - log.info('Setting up the master communication server') - self.clients.bind(self.uri) - - self.workers.bind(self.w_uri) - - while True: - try: - zmq.device(zmq.QUEUE, self.clients, self.workers) - except zmq.ZMQError as exc: - if exc.errno == errno.EINTR: - continue - raise exc - - -class SaltZmqCrypticleSetup(ioflo.base.deeding.Deed): - ''' - Setup the crypticle for the salt zmq publisher behavior - - do salt zmq crypticle setup at enter - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'aes': salt.utils.stringutils.to_str('.salt.var.zmq.aes'), - 'crypticle': salt.utils.stringutils.to_str('.salt.var.zmq.crypticle')} - - def action(self): - ''' - Initializes zmq - Put here so only runs initialization if we want multi-headed master - - ''' - self.crypticle.value = salt.crypt.Crypticle( - self.opts.value, - self.opts.value.get('aes')) - - -class SaltZmqPublisher(ioflo.base.deeding.Deed): - ''' - The zeromq publisher - - do salt zmq publisher - - Must run the deed - - do salt zmq publisher setup - - before this deed - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'publish': salt.utils.stringutils.to_str('.salt.var.publish'), - 'zmq_behavior': salt.utils.stringutils.to_str('.salt.etc.zmq_behavior'), - 'aes': salt.utils.stringutils.to_str('.salt.var.zmq.aes'), - 'crypticle': salt.utils.stringutils.to_str('.salt.var.zmq.crypticle')} - - def _prepare(self): - ''' - Set up tracking value(s) - ''' - if not zmq: - return - self.created = False - self.serial = salt.payload.Serial(self.opts.value) - - def action(self): - ''' - Create the publish port if it is not available and then publish the - messages on it - ''' - if not self.zmq_behavior: - return - if not self.created: - self.context = zmq.Context(1) - self.pub_sock = self.context.socket(zmq.PUB) - # if 2.1 >= zmq < 3.0, we only have one HWM setting - try: - self.pub_sock.setsockopt(zmq.HWM, self.opts.value.get('pub_hwm', 1000)) - # in zmq >= 3.0, there are separate send and receive HWM settings - except AttributeError: - self.pub_sock.setsockopt(zmq.SNDHWM, self.opts.value.get('pub_hwm', 1000)) - self.pub_sock.setsockopt(zmq.RCVHWM, self.opts.value.get('pub_hwm', 1000)) - if self.opts.value['ipv6'] is True and hasattr(zmq, 'IPV4ONLY'): - # IPv6 sockets work for both IPv6 and IPv4 addresses - self.pub_sock.setsockopt(zmq.IPV4ONLY, 0) - self.pub_sock.setsockopt(zmq.BACKLOG, self.opts.get('zmq_backlog', 1000)) - self.pub_uri = 'tcp://{interface}:{publish_port}'.format(**self.opts.value) - log.info('Starting the Salt ZeroMQ Publisher on %s', self.pub_uri) - self.pub_sock.bind(self.pub_uri) - self.created = True - # Don't pop the publish messages! The raet behavior still needs them - try: - for package in self.publish.value: - payload = {'enc': 'aes'} - payload['load'] = self.crypticle.value.dumps(package['return']['pub']) - if self.opts.value['sign_pub_messages']: - master_pem_path = os.path.join(self.opts.value['pki_dir'], 'master.pem') - log.debug('Signing data packet for publish') - payload['sig'] = salt.crypt.sign_message(master_pem_path, payload['load']) - - send_payload = self.serial.dumps(payload) - if self.opts.value['zmq_filtering']: - # if you have a specific topic list, use that - if package['return']['pub']['tgt_type'] == 'list': - for topic in package['return']['pub']['tgt']: - # zmq filters are substring match, hash the topic - # to avoid collisions - htopic = hashlib.sha1(topic).hexdigest() - self.pub_sock.send(htopic, flags=zmq.SNDMORE) - self.pub_sock.send(send_payload) - # otherwise its a broadcast - else: - self.pub_sock.send('broadcast', flags=zmq.SNDMORE) - self.pub_sock.send(send_payload) - else: - self.pub_sock.send(send_payload) - except zmq.ZMQError as exc: - if exc.errno == errno.EINTR: - return - raise exc - - -class SaltZmqWorker(ioflo.base.deeding.Deed): - ''' - The zeromq behavior for the workers - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'key': salt.utils.stringutils.to_str('.salt.access_keys'), - 'aes': salt.utils.stringutils.to_str('.salt.var.zmq.aes')} - - def _prepare(self): - ''' - Create the initial seting value for the worker - ''' - self.created = False - - def action(self): - ''' - Create the master MWorker if it is not present, then iterate over the - connection with the ioflo sequence - ''' - if not self.created: - crypticle = salt.crypt.Crypticle(self.opts.value, self.aes.value) - self.worker = salt.master.FloMWorker( - self.opts.value, - self.key.value, - ) - self.worker.setup() - self.created = True - log.info('Started ZMQ worker') - self.worker.handle_request() diff --git a/salt/daemons/salting.py b/salt/daemons/salting.py deleted file mode 100644 index 80a27dfb299..00000000000 --- a/salt/daemons/salting.py +++ /dev/null @@ -1,279 +0,0 @@ -# -*- coding: utf-8 -*- -''' -salting.py module of salt specific interfaces to raet - -''' -from __future__ import absolute_import, print_function, unicode_literals -# pylint: skip-file -# pylint: disable=W0611 - -# Import Python libs -import os - -# Import ioflo libs -from ioflo.aid.odicting import odict - -from ioflo.base.consoling import getConsole -console = getConsole() - -from raet import raeting, nacling -from raet.keeping import Keep - -from salt.key import RaetKey - -import salt.utils.kinds as kinds - - -class SaltKeep(Keep): - ''' - RAET protocol estate on road data persistence for a given estate - road specific data - - road/ - keep/ - stackname/ - local/ - estate.ext - remote/ - estate.name.ext - estate.name.ext - ''' - LocalFields = ['name', 'uid', 'ha', 'iha', 'natted', 'fqdn', 'dyned', 'sid', - 'puid', 'aha', 'role', 'sighex','prihex'] - LocalDumpFields = ['name', 'uid', 'ha', 'iha', 'natted', 'fqdn', 'dyned', 'sid', - 'puid', 'aha', 'role'] - RemoteFields = ['name', 'uid', 'fuid', 'ha', 'iha', 'natted', 'fqdn', 'dyned', - 'sid', 'main', 'kind', 'joined', - 'role', 'acceptance', 'verhex', 'pubhex'] - RemoteDumpFields = ['name', 'uid', 'fuid', 'ha', 'iha', 'natted', 'fqdn', 'dyned', - 'sid', 'main', 'kind', 'joined', 'role'] - Auto = raeting.AutoMode.never.value #auto accept - - def __init__(self, opts, prefix='estate', basedirpath='', auto=None, **kwa): - ''' - Setup RoadKeep instance - ''' - basedirpath = basedirpath or os.path.join(opts['cache_dir'], 'raet') - super(SaltKeep, self).__init__(prefix=prefix, basedirpath=basedirpath, **kwa) - self.auto = (auto if auto is not None else - (raeting.AutoMode.always.value if opts['open_mode'] else - (raeting.AutoMode.once.value if opts['auto_accept'] else - raeting.AutoMode.never.value))) - self.saltRaetKey = RaetKey(opts) - - def clearAllDir(self): - ''' - Clear all keep directories - ''' - super(SaltKeep, self).clearAllDir() - self.clearRoleDir() - - def clearRoleDir(self): - ''' - Clear the Role directory - ''' - self.saltRaetKey.delete_pki_dir() - - def loadLocalRoleData(self): - ''' - Load and return the role data - ''' - keydata = self.saltRaetKey.read_local() - if not keydata: - keydata = odict([('sign', None), ('priv', None)]) - data = odict([('sighex', keydata['sign']), - ('prihex', keydata['priv'])]) - return data - - def clearLocalRoleData(self): - ''' - Clear the local file - ''' - self.saltRaetKey.delete_local() - - def clearLocalRoleDir(self): - ''' - Clear the Local Role directory - ''' - self.saltRaetKey.delete_pki_dir() - - def loadLocalData(self): - ''' - Load and Return the data from the local estate - ''' - data = super(SaltKeep, self).loadLocalData() - if not data: - return None - roleData = self.loadLocalRoleData() # if not present defaults None values - data.update([('sighex', roleData.get('sighex')), - ('prihex', roleData.get('prihex'))]) - return data - - def loadRemoteData(self, name): - ''' - Load and Return the data from the remote file - ''' - data = super(SaltKeep, self).loadRemoteData(name) - if not data: - return None - - mid = data['role'] - for status in [acceptance.name for acceptance in Acceptance]: - keydata = self.saltRaetKey.read_remote(mid, status) - if keydata: - break - - if not keydata: - data.update([('acceptance', None), - ('verhex', None), - ('pubhex', None)]) - else: - data.update(acceptance=raeting.Acceptance[status].value, - verhex=keydata['verify'], - pubhex=keydata['pub']) - - return data - - def loadAllRemoteData(self): - ''' - Load and Return the data from the all the remote estate files - ''' - keeps = super(SaltKeep, self).loadAllRemoteData() - for name, data in keeps.items(): - keeps[name].update([('acceptance', None), - ('verhex', None), - ('pubhex', None)]) - - for status, mids in self.saltRaetKey.list_keys().items(): - for mid in mids: - keydata = self.saltRaetKey.read_remote(mid, status) - if keydata: - for name, data in keeps.items(): - if data['role'] == mid: - keeps[name].update( - [('acceptance', raeting.Acceptance[status].value), - ('verhex', keydata['verify']), - ('pubhex', keydata['pub'])]) - return keeps - - def clearRemoteRoleData(self, role): - ''' - Clear data from the role data file - ''' - self.saltRaetKey.delete_key(role) #now delete role key file - - def clearAllRemoteRoleData(self): - ''' - Remove all the role data files - ''' - self.saltRaetKey.delete_all() - - def clearRemoteRoleDir(self): - ''' - Clear the Remote Role directory - ''' - self.saltRaetKey.delete_pki_dir() - - def dumpLocal(self, local): - ''' - Dump local estate - ''' - data = odict([ - ('name', local.name), - ('uid', local.uid), - ('ha', local.ha), - ('iha', local.iha), - ('natted', local.natted), - ('fqdn', local.fqdn), - ('dyned', local.dyned), - ('sid', local.sid), - ('puid', local.stack.puid), - ('aha', local.stack.aha), - ('role', local.role), - ]) - if self.verifyLocalData(data, localFields =self.LocalDumpFields): - self.dumpLocalData(data) - - self.saltRaetKey.write_local(local.priver.keyhex, local.signer.keyhex) - - def dumpRemote(self, remote): - ''' - Dump remote estate - ''' - data = odict([ - ('name', remote.name), - ('uid', remote.uid), - ('fuid', remote.fuid), - ('ha', remote.ha), - ('iha', remote.iha), - ('natted', remote.natted), - ('fqdn', remote.fqdn), - ('dyned', remote.dyned), - ('sid', remote.sid), - ('main', remote.main), - ('kind', remote.kind), - ('joined', remote.joined), - ('role', remote.role), - ]) - if self.verifyRemoteData(data, remoteFields=self.RemoteDumpFields): - self.dumpRemoteData(data, remote.name) - - if remote.pubber.keyhex and remote.verfer.keyhex: - # kludge to persist the keys since no way to write - self.saltRaetKey.status(remote.role, - remote.pubber.keyhex, - remote.verfer.keyhex) - - def statusRemote(self, remote, dump=True): - ''' - Calls .statusRole on remote role and keys and updates remote.acceptance - dump indicates if statusRole should update persisted values when - appropriate. - - Returns status - Where status is acceptance status of role and keys - and has value from raeting.acceptances - ''' - status = self.statusRole(role=remote.role, - verhex=remote.verfer.keyhex, - pubhex=remote.pubber.keyhex, - dump=dump) - - remote.acceptance = status - - return status - - def statusRole(self, role, verhex, pubhex, dump=True): - ''' - Returns status - - Where status is acceptance status of role and keys - and has value from raeting.acceptances - ''' - status = raeting.Acceptance[self.saltRaetKey.status(role, - pubhex, - verhex)].value - - return status - - def rejectRemote(self, remote): - ''' - Set acceptance status to rejected - ''' - mid = remote.role - self.saltRaetKey.reject(match=mid, include_accepted=True) - remote.acceptance = raeting.Acceptance.rejected.value - - def pendRemote(self, remote): - ''' - Set acceptance status to pending - ''' - pass - - def acceptRemote(self, remote): - ''' - Set acceptance status to accepted - ''' - mid = remote.role - self.saltRaetKey.accept(match=mid, include_rejected=True) - remote.acceptance = raeting.Acceptance.accepted.value diff --git a/salt/daemons/test/__init__.py b/salt/daemons/test/__init__.py deleted file mode 100644 index f98b8fd42f7..00000000000 --- a/salt/daemons/test/__init__.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- - -''' -salt daemons raet unit test package - -To run the unittests: - -from salt.daemons import test -test.run() - -''' -from __future__ import absolute_import, print_function, unicode_literals -import sys -# pylint: disable=blacklisted-import -if sys.version_info < (2, 7): - import unittest2 as unittest -else: - import unittest -# pylint: enable=blacklisted-import -import os - - -from ioflo.base.consoling import getConsole -console = getConsole() -console.reinit(verbosity=console.Wordage.concise) - - -def run(start=None): - ''' - Run unittests starting at directory given by start - Default start is the location of the raet package - ''' - top = os.path.dirname(os.path.dirname(os.path.abspath( - sys.modules.get(__name__).__file__))) - - if not start: - start = top - - console.terse("\nRunning all salt.daemons unit tests in '{0}', starting at '{1}'\n".format(top, start)) - loader = unittest.TestLoader() - suite = loader.discover(start, 'test_*.py', top) - unittest.TextTestRunner(verbosity=2).run(suite) - - -if __name__ == "__main__": - run() diff --git a/salt/daemons/test/master.flo b/salt/daemons/test/master.flo deleted file mode 100644 index 728155e7c40..00000000000 --- a/salt/daemons/test/master.flo +++ /dev/null @@ -1,26 +0,0 @@ -# Raet Test FloScript - -house master - -init .salt.road.manor.local to role "master" host "" port 7530 main true - -framer masterudpstack be active first start - frame start - do salt raet cleanup at enter - do salt raet road stack setup per inode ".salt.road.manor" at enter - bid start service - do raet road stack closer per inode ".salt.road.manor." at exit - -framer printer be active first start - frame start - do salt raet road stack printer per inode ".salt.road.manor." - timeout 20 - - frame abort - bid stop all - -framer service be inactive first start - frame start - do salt raet road stack service - - diff --git a/salt/daemons/test/minion.flo b/salt/daemons/test/minion.flo deleted file mode 100644 index 02a4e465ddb..00000000000 --- a/salt/daemons/test/minion.flo +++ /dev/null @@ -1,66 +0,0 @@ -# Raet Test FloScript - -house minion - -init .salt.road.manor.local to name "minion" host "" port 7531 main false - -framer minionudpstack be active first start - frame start - enter - do salt raet road stack setup per inode ".salt.road.manor" - exit - do salt raet road stack closer per inode ".salt.road.manor." - -framer bootstrap be active first setup - frame setup - enter - do salt raet road usher minion setup per inode ".salt.road.manor." - go join - - frame join - print Joining... - enter - do salt raet road stack joiner per inode ".salt.road.manor." - recur - do raet road stack joined per inode ".salt.road.manor." - - go next if joined in .salt.road.manor.status - go abort if elapsed >= 10 - - frame joined - print Joined - go next - - frame allow - print Allowing... - enter - do salt raet road stack allower per inode ".salt.road.manor." - recur - do salt raet road stack allowed per inode ".salt.road.manor." - - go next if allowed in .salt.road.manor.status - go abort if elapsed >= 5 - - - frame allowed - print Allowed - go next - - frame message - print Messaging... - enter - do raet road stack messenger with contents "Minion 1 Hello" code 15 \ - per inode ".salt.road.manor." - go next - - frame idle - print Idling... - do raet road stack idled per inode ".salt.road.manor." - go abort if idled in .salt.road.manor.status - - frame abort - bid stop all - -framer service be active first start - frame start - do salt raet road stack service diff --git a/salt/daemons/test/plan/__init__.py b/salt/daemons/test/plan/__init__.py deleted file mode 100644 index 783a30a91e3..00000000000 --- a/salt/daemons/test/plan/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import, print_function, unicode_literals -from . import actors - -__all__ = ['actors'] diff --git a/salt/daemons/test/plan/actors.py b/salt/daemons/test/plan/actors.py deleted file mode 100644 index 25328bf05d9..00000000000 --- a/salt/daemons/test/plan/actors.py +++ /dev/null @@ -1,829 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Test behaviors used by test plans -''' -from __future__ import absolute_import, print_function, unicode_literals -import os -import stat -import time -from collections import deque - -# Import ioflo libs -import ioflo.base.deeding -from ioflo.aid.odicting import odict - -from ioflo.base.consoling import getConsole -console = getConsole() - -from raet import raeting, nacling -from raet.lane.stacking import LaneStack -from raet.lane.yarding import RemoteYard -from raet.road.estating import RemoteEstate -from raet.road.stacking import RoadStack -from raet.stacking import Stack - -import salt.utils.stringutils -from salt.daemons import salting -from salt.utils.event import tagify - - -class DeedTestWrapper(object): - def assertTrue(self, condition): - if not condition: - self.failure.value = 'Fail' - raise Exception("Test Failed") - - -def createStack(ip): - stack = Stack() - stack.ha = (ip, '1234') - return stack - - -class TestOptsSetup(ioflo.base.deeding.Deed): - ''' - Setup opts share - ''' - Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts')} - - def action(self): - ''' - Register presence requests - Iterate over the registered presence yards and fire! - ''' - pkiDirpath = os.path.join('/tmp', 'raet', 'test', self.role, 'pki') - if not os.path.exists(pkiDirpath): - os.makedirs(pkiDirpath) - - acceptedDirpath = os.path.join(pkiDirpath, 'accepted') - if not os.path.exists(acceptedDirpath): - os.makedirs(acceptedDirpath) - - pendingDirpath = os.path.join(pkiDirpath, 'pending') - if not os.path.exists(pendingDirpath): - os.makedirs(pendingDirpath) - - rejectedDirpath = os.path.join(pkiDirpath, 'rejected') - if not os.path.exists(rejectedDirpath): - os.makedirs(rejectedDirpath) - - localFilepath = os.path.join(pkiDirpath, 'local.key') - if os.path.exists(localFilepath): - mode = os.stat(localFilepath).st_mode - print(mode) - os.chmod(localFilepath, mode | stat.S_IWUSR | stat.S_IRUSR) - mode = os.stat(localFilepath).st_mode - print(mode) - - cacheDirpath = os.path.join('/tmp/raet', 'cache', self.role) - if not os.path.exists(cacheDirpath): - os.makedirs(cacheDirpath) - - sockDirpath = os.path.join('/tmp/raet', 'sock', self.role) - if not os.path.exists(sockDirpath): - os.makedirs(sockDirpath) - - self.opts.value = dict( - id=self.role, - __role=self.role, - ioflo_period=0.1, - ioflo_realtime=True, - ioflo_verbose=2, - interface="", - raet_port=self.raet_port, - transport='raet', - client_acl=dict(), - publisher_acl=dict(), - pki_dir=pkiDirpath, - sock_dir=sockDirpath, - cachedir=cacheDirpath, - open_mode=True, - auto_accept=True, - ) - - name = "{0}_{1}".format(self.role, self.role) - basedirpath = os.path.abspath(os.path.join(cacheDirpath, 'raet')) - keep = salting.SaltKeep(opts=self.opts.value, - basedirpath=basedirpath, - stackname=name) - keep.clearLocalData() - keep.clearLocalRoleData() - keep.clearAllRemoteData() - keep.clearAllRemoteRoleData() - - -class TestOptsSetupMaster(TestOptsSetup): - - def action(self): - self.role = 'master' - self.raet_port = raeting.RAET_PORT - super(TestOptsSetupMaster, self).action() - - -class TestOptsSetupMinion(TestOptsSetup): - - def action(self): - self.role = 'minion' - self.raet_port = raeting.RAET_TEST_PORT - super(TestOptsSetupMinion, self).action() - - -def serviceRoads(stacks, duration=1.0): - ''' - Utility method to service queues for list of stacks. Call from test method. - ''' - start = time.time() - while start + duration > time.time(): - for stack in stacks: - stack.serviceAll() - if all([not stack.transactions for stack in stacks]): - console.terse("Service stacks done normally\n") - break - time.sleep(0.05) - for stack in stacks: - console.terse("Stack {0} remotes: {1}\n".format(stack.name, stack.nameRemotes)) - console.terse("Service stacks exit\n") - - -def serviceLanes(stacks, duration=1.0): - ''' - Utility method to service queues for list of stacks. Call from test method. - ''' - start = time.time() - while start + duration > time.time(): - for stack in stacks: - stack.serviceAll() - if all([not stack.txMsgs for stack in stacks]): - console.terse("Service stacks done normally\n") - break - time.sleep(0.05) - for stack in stacks: - console.terse("Stack {0} remotes: {1}\n".format(stack.name, stack.nameRemotes)) - console.terse("Service stacks exit\n") - - -class PresenterTestSetup(ioflo.base.deeding.Deed): - ''' - Setup shares for presence tests - ''' - Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), - 'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'), - 'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'alloweds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.alloweds'), - 'ival': odict()}, - 'aliveds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.aliveds'), - 'ival': odict()}, - 'reapeds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.reapeds'), - 'ival': odict()}, - 'availables': {'ipath': salt.utils.stringutils.to_str( - '.salt.var.presence.availables'), - 'ival': set()}} - - def action(self): - - self.presence_req.value = deque() - self.availables.value = set() - self.alloweds.value = odict() - self.aliveds.value = odict() - self.reapeds.value = odict() - - # Create event stack - name = 'event' + nacling.uuid(size=18) - lanename = self.lane_stack.value.local.lanename - sock_dir = self.lane_stack.value.local.dirpath - ryn = 'manor' - console.terse("Create stack: name = {0}, lanename = {1}, sock_dir = {2}\n". - format(name, lanename, sock_dir)) - stack = LaneStack( - name=name, - lanename=lanename, - sockdirpath=sock_dir) - stack.addRemote(RemoteYard(stack=stack, - lanename=lanename, - name=ryn, - dirpath=sock_dir)) - self.event_stack.value = stack - - route = {'dst': (None, ryn, 'presence_req'), - 'src': (None, stack.local.name, None)} - msg = {'route': route} - stack.transmit(msg, stack.nameRemotes[ryn].uid) - serviceLanes([stack, self.lane_stack.value]) - - -class PresenterTestCleanup(ioflo.base.deeding.Deed): - ''' - Clean up after a test - ''' - Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), - 'alloweds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.alloweds'), - 'ival': odict()}, - 'aliveds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.aliveds'), - 'ival': odict()}, - 'reapeds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.reapeds'), - 'ival': odict()}, - 'availables': {'ipath': salt.utils.stringutils.to_str( - '.salt.var.presence.availables'), - 'ival': set()}} - - def action(self): - - self.presence_req.value = deque() - self.availables.value = set() - self.alloweds.value = odict() - self.aliveds.value = odict() - self.reapeds.value = odict() - - -class TestPresenceAvailable(ioflo.base.deeding.Deed): - Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), - 'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'aliveds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.aliveds'), - 'ival': odict()}, - 'availables': {'ipath': salt.utils.stringutils.to_str( - '.salt.var.presence.availables'), - 'ival': set()}} - - def action(self): - ''' - Test Presenter 'available' request (A1, B*) - ''' - console.terse("{0}\n".format(self.action.__doc__)) - - # Prepare - # add available minions - self.availables.value.add('alpha') - self.availables.value.add('beta') - self.aliveds.value['alpha'] = createStack('1.1.1.1') - self.aliveds.value['beta'] = createStack('1.2.3.4') - # add presence request - testStack = self.event_stack.value - presenceReq = self.presence_req.value - ryn = 'manor' - # general available request format - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'available'}}) - # missing 'data', fallback to available - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}}) - # missing 'state' in 'data', fallback to available - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {}}) - # requested None state, fallback to available - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': None}}) - # requested 'present' state that is alias for available - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'present'}}) - - -class TestPresenceAvailableCheck(ioflo.base.deeding.Deed, DeedTestWrapper): - Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'failure': salt.utils.stringutils.to_str('.meta.failure')} - - def action(self): - testStack = self.event_stack.value - self.assertTrue(len(testStack.rxMsgs) == 0) - testStack.serviceAll() - self.assertTrue(len(testStack.rxMsgs) == 5) - - tag = tagify('present', 'presence') - while testStack.rxMsgs: - msg, sender = testStack.rxMsgs.popleft() - self.assertTrue(msg == {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'present': {'alpha': '1.1.1.1', - 'beta': '1.2.3.4'}}}) - - -class TestPresenceJoined(ioflo.base.deeding.Deed): - Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), - 'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'alloweds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.alloweds'), - 'ival': odict()}} - - def action(self): - ''' - Test Presenter 'joined' request (A2) - ''' - console.terse("{0}\n".format(self.action.__doc__)) - - # Prepare - # add joined minions - self.alloweds.value['alpha'] = createStack('1.1.1.1') - self.alloweds.value['beta'] = createStack('1.2.3.4') - # add presence request - testStack = self.event_stack.value - presenceReq = self.presence_req.value - ryn = 'manor' - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'joined'}}) - - -class TestPresenceJoinedCheck(ioflo.base.deeding.Deed, DeedTestWrapper): - Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'failure': salt.utils.stringutils.to_str('.meta.failure')} - - def action(self): - testStack = self.event_stack.value - self.assertTrue(len(testStack.rxMsgs) == 0) - testStack.serviceAll() - self.assertTrue(len(testStack.rxMsgs) == 1) - - tag = tagify('present', 'presence') - msg, sender = testStack.rxMsgs.popleft() - self.assertTrue(msg == {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'joined': {'alpha': '1.1.1.1', - 'beta': '1.2.3.4'}}}) - - -class TestPresenceAllowed(ioflo.base.deeding.Deed): - Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), - 'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'alloweds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.alloweds'), - 'ival': odict()}} - - def action(self): - ''' - Test Presenter 'allowed' request (A3) - ''' - console.terse("{0}\n".format(self.action.__doc__)) - - # Prepare - # add allowed minions - self.alloweds.value['alpha'] = createStack('1.1.1.1') - self.alloweds.value['beta'] = createStack('1.2.3.4') - # add presence request - testStack = self.event_stack.value - presenceReq = self.presence_req.value - ryn = 'manor' - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'allowed'}}) - - -class TestPresenceAllowedCheck(ioflo.base.deeding.Deed, DeedTestWrapper): - Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'failure': salt.utils.stringutils.to_str('.meta.failure')} - - def action(self): - testStack = self.event_stack.value - self.assertTrue(len(testStack.rxMsgs) == 0) - testStack.serviceAll() - self.assertTrue(len(testStack.rxMsgs) == 1) - - tag = tagify('present', 'presence') - msg, sender = testStack.rxMsgs.popleft() - self.assertTrue(msg == {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'allowed': {'alpha': '1.1.1.1', - 'beta': '1.2.3.4'}}}) - - -class TestPresenceAlived(ioflo.base.deeding.Deed): - Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), - 'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'aliveds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.aliveds'), - 'ival': odict()}} - - def action(self): - ''' - Test Presenter 'alived' request (A4) - ''' - console.terse("{0}\n".format(self.action.__doc__)) - - # Prepare - # add alived minions - self.aliveds.value['alpha'] = createStack('1.1.1.1') - self.aliveds.value['beta'] = createStack('1.2.3.4') - # add presence request - testStack = self.event_stack.value - presenceReq = self.presence_req.value - ryn = 'manor' - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'alived'}}) - - -class TestPresenceAlivedCheck(ioflo.base.deeding.Deed, DeedTestWrapper): - Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'failure': salt.utils.stringutils.to_str('.meta.failure')} - - def action(self): - testStack = self.event_stack.value - self.assertTrue(len(testStack.rxMsgs) == 0) - testStack.serviceAll() - self.assertTrue(len(testStack.rxMsgs) == 1) - - tag = tagify('present', 'presence') - msg, sender = testStack.rxMsgs.popleft() - self.assertTrue(msg == {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'alived': {'alpha': '1.1.1.1', - 'beta': '1.2.3.4'}}}) - - -class TestPresenceReaped(ioflo.base.deeding.Deed): - Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), - 'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'reapeds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.reapeds'), - 'ival': odict()}} - - def action(self): - ''' - Test Presenter 'reaped' request (A5) - ''' - console.terse("{0}\n".format(self.action.__doc__)) - - # Prepare - # add reaped minions - self.reapeds.value['alpha'] = createStack('1.1.1.1') - self.reapeds.value['beta'] = createStack('1.2.3.4') - # add presence request - testStack = self.event_stack.value - presenceReq = self.presence_req.value - ryn = 'manor' - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'reaped'}}) - - -class TestPresenceReapedCheck(ioflo.base.deeding.Deed, DeedTestWrapper): - Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'failure': salt.utils.stringutils.to_str('.meta.failure')} - - def action(self): - testStack = self.event_stack.value - self.assertTrue(len(testStack.rxMsgs) == 0) - testStack.serviceAll() - self.assertTrue(len(testStack.rxMsgs) == 1) - - tag = tagify('present', 'presence') - msg, sender = testStack.rxMsgs.popleft() - self.assertTrue(msg == {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'reaped': {'alpha': '1.1.1.1', - 'beta': '1.2.3.4'}}}) - - -class TestPresenceNoRequest(ioflo.base.deeding.Deed): - Ioinits = {} - - def action(self): - ''' - Test Presenter with no requests (C1) - ''' - console.terse("{0}\n".format(self.action.__doc__)) - - -class TestPresenceNoRequestCheck(ioflo.base.deeding.Deed, DeedTestWrapper): - Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'failure': salt.utils.stringutils.to_str('.meta.failure')} - - def action(self): - testStack = self.event_stack.value - self.assertTrue(len(testStack.rxMsgs) == 0) - testStack.serviceAll() - self.assertTrue(len(testStack.rxMsgs) == 0) - - -class TestPresenceUnknownSrc(ioflo.base.deeding.Deed, DeedTestWrapper): - Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), - 'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'failure': salt.utils.stringutils.to_str('.meta.failure')} - - def action(self): - ''' - Test Presenter handles request from unknown (disconnected) source (C2) - ''' - console.terse("{0}\n".format(self.action.__doc__)) - - # Prepare - # add presence request - testStack = self.event_stack.value - presenceReq = self.presence_req.value - ryn = 'manor' - name = 'unknown_name' - self.assertTrue(name != testStack.local.name) - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, name, None)}}) - - -class TestPresenceUnknownSrcCheck(ioflo.base.deeding.Deed, DeedTestWrapper): - Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'failure': salt.utils.stringutils.to_str('.meta.failure')} - - def action(self): - testStack = self.event_stack.value - self.assertTrue(len(testStack.rxMsgs) == 0) - testStack.serviceAll() - self.assertTrue(len(testStack.rxMsgs) == 0) - - -class TestPresenceAvailableNoMinions(ioflo.base.deeding.Deed): - Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), - 'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack')} - - def action(self): - ''' - Test Presenter 'available' request with no minions in the state (D1) - ''' - console.terse("{0}\n".format(self.action.__doc__)) - - # Prepare - # add presence request - testStack = self.event_stack.value - presenceReq = self.presence_req.value - ryn = 'manor' - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'available'}}) - - -class TestPresenceAvailableNoMinionsCheck(ioflo.base.deeding.Deed, DeedTestWrapper): - Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'failure': salt.utils.stringutils.to_str('.meta.failure')} - - def action(self): - testStack = self.event_stack.value - self.assertTrue(len(testStack.rxMsgs) == 0) - testStack.serviceAll() - self.assertTrue(len(testStack.rxMsgs) == 1) - - tag = tagify('present', 'presence') - while testStack.rxMsgs: - msg, sender = testStack.rxMsgs.popleft() - self.assertTrue(msg == {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'present': {}}}) - - -class TestPresenceAvailableOneMinion(ioflo.base.deeding.Deed): - Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), - 'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'aliveds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.aliveds'), - 'ival': odict()}, - 'availables': {'ipath': salt.utils.stringutils.to_str( - '.salt.var.presence.availables'), - 'ival': set()}} - - def action(self): - ''' - Test Presenter 'available' request with one minions in the state (D2) - ''' - console.terse("{0}\n".format(self.action.__doc__)) - - # Prepare - # add available minions - self.availables.value.add('alpha') - self.aliveds.value['alpha'] = createStack('1.1.1.1') - # add presence request - testStack = self.event_stack.value - presenceReq = self.presence_req.value - ryn = 'manor' - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'available'}}) - - -class TestPresenceAvailableOneMinionCheck(ioflo.base.deeding.Deed, DeedTestWrapper): - Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'failure': salt.utils.stringutils.to_str('.meta.failure')} - - def action(self): - testStack = self.event_stack.value - self.assertTrue(len(testStack.rxMsgs) == 0) - testStack.serviceAll() - self.assertTrue(len(testStack.rxMsgs) == 1) - - tag = tagify('present', 'presence') - while testStack.rxMsgs: - msg, sender = testStack.rxMsgs.popleft() - self.assertTrue(msg == {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'present': {'alpha': '1.1.1.1'}}}) - - -class TestPresenceAvailableUnknownIp(ioflo.base.deeding.Deed): - Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), - 'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'aliveds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.aliveds'), - 'ival': odict()}, - 'availables': {'ipath': salt.utils.stringutils.to_str( - '.salt.var.presence.availables'), - 'ival': set()}} - - def action(self): - ''' - Test Presenter 'available' request with one minions in the state (D3) - ''' - console.terse("{0}\n".format(self.action.__doc__)) - - # Prepare - # add available minions - self.availables.value.add('alpha') - self.availables.value.add('beta') - self.availables.value.add('gamma') - self.aliveds.value['alpha'] = createStack('1.1.1.1') - self.aliveds.value['delta'] = createStack('1.2.3.4') - # add presence request - testStack = self.event_stack.value - presenceReq = self.presence_req.value - ryn = 'manor' - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'available'}}) - - -class TestPresenceAvailableUnknownIpCheck(ioflo.base.deeding.Deed, DeedTestWrapper): - Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'failure': salt.utils.stringutils.to_str('.meta.failure')} - - def action(self): - testStack = self.event_stack.value - self.assertTrue(len(testStack.rxMsgs) == 0) - testStack.serviceAll() - self.assertTrue(len(testStack.rxMsgs) == 1) - - tag = tagify('present', 'presence') - while testStack.rxMsgs: - msg, sender = testStack.rxMsgs.popleft() - self.assertTrue(msg == {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'present': {'alpha': '1.1.1.1', - 'beta': None, - 'gamma': None}}}) - - -class TestPresenceAllowedNoMinions(ioflo.base.deeding.Deed): - Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), - 'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack')} - - def action(self): - ''' - Test Presenter 'allowed' request with no minions in the state (D4) - ''' - console.terse("{0}\n".format(self.action.__doc__)) - - # Prepare - # add presence request - testStack = self.event_stack.value - presenceReq = self.presence_req.value - ryn = 'manor' - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'allowed'}}) - - -class TestPresenceAllowedNoMinionsCheck(ioflo.base.deeding.Deed, DeedTestWrapper): - Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'failure': salt.utils.stringutils.to_str('.meta.failure')} - - def action(self): - testStack = self.event_stack.value - self.assertTrue(len(testStack.rxMsgs) == 0) - testStack.serviceAll() - self.assertTrue(len(testStack.rxMsgs) == 1) - - tag = tagify('present', 'presence') - msg, sender = testStack.rxMsgs.popleft() - self.assertTrue(msg == {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'allowed': {}}}) - - -class TestPresenceAllowedOneMinion(ioflo.base.deeding.Deed): - Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), - 'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'alloweds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.alloweds'), - 'ival': odict()}} - - def action(self): - ''' - Test Presenter 'allowed' request with one minion in the state (D5) - ''' - console.terse("{0}\n".format(self.action.__doc__)) - - # Prepare - # add allowed minion - self.alloweds.value['alpha'] = createStack('1.1.1.1') - # add presence request - testStack = self.event_stack.value - presenceReq = self.presence_req.value - ryn = 'manor' - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'allowed'}}) - - -class TestPresenceAllowedOneMinionCheck(ioflo.base.deeding.Deed, DeedTestWrapper): - Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'), - 'failure': salt.utils.stringutils.to_str('.meta.failure')} - - def action(self): - testStack = self.event_stack.value - self.assertTrue(len(testStack.rxMsgs) == 0) - testStack.serviceAll() - self.assertTrue(len(testStack.rxMsgs) == 1) - - tag = tagify('present', 'presence') - msg, sender = testStack.rxMsgs.popleft() - self.assertTrue(msg == {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'allowed': {'alpha': '1.1.1.1'}}}) - - -class StatsMasterTestSetup(ioflo.base.deeding.Deed): - ''' - Setup shares for stats tests - ''' - Ioinits = {'stats_req': salt.utils.stringutils.to_str('.salt.stats.event_req'), - 'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'), - 'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'), - 'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack')} - - def action(self): - - self.stats_req.value = deque() - - # Create event stack - name = 'event' + nacling.uuid(size=18) - lanename = self.lane_stack.value.local.lanename - sock_dir = self.lane_stack.value.local.dirpath - ryn = 'manor' - console.terse("Create stack: name = {0}, lanename = {1}, sock_dir = {2}\n". - format(name, lanename, sock_dir)) - stack = LaneStack( - name=name, - lanename=lanename, - sockdirpath=sock_dir) - stack.addRemote(RemoteYard(stack=stack, - lanename=lanename, - name=ryn, - dirpath=sock_dir)) - self.event_stack.value = stack - - route = {'dst': (None, ryn, 'stats_req'), - 'src': (None, stack.local.name, None)} - msg = {'route': route} - stack.transmit(msg, stack.nameRemotes[ryn].uid) - serviceLanes([stack, self.lane_stack.value]) - - -class StatsMinionTestSetup(ioflo.base.deeding.Deed): - ''' - Setup shares for stats tests - ''' - Ioinits = {'stats_req': salt.utils.stringutils.to_str('.salt.stats.event_req'), - 'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'), - 'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'), - 'event_stack': salt.utils.stringutils.to_str('.salt.test.road.stack')} - - def action(self): - - self.stats_req.value = deque() - - minionStack = self.road_stack.value - - # Create Master Stack - self.store.stamp = 0.0 - masterStack = RoadStack(store=self.store, - name='master', - ha=('', raeting.RAET_PORT), - role='master', - main=True, - cleanremote=True, - period=3.0, - offset=0.5) - self.event_stack.value = masterStack - - minionRemoteMaster = RemoteEstate(stack=minionStack, - fuid=0, - sid=0, - ha=masterStack.local.ha) - minionStack.addRemote(minionRemoteMaster) - - # Make life easier - masterStack.keep.auto = raeting.AutoMode.always.value - minionStack.keep.auto = raeting.AutoMode.always.value - - minionStack.join(minionRemoteMaster.uid) - serviceRoads([minionStack, masterStack]) - minionStack.allow(minionRemoteMaster.uid) - serviceRoads([minionStack, masterStack]) diff --git a/salt/daemons/test/plan/box1.flo b/salt/daemons/test/plan/box1.flo deleted file mode 100644 index 706290ac657..00000000000 --- a/salt/daemons/test/plan/box1.flo +++ /dev/null @@ -1,118 +0,0 @@ -# Test for presence functionality - -house presence - -framer presenter be active first setup - frame setup - do test opts setup master - do salt raet manor lane setup per inode ".salt.lane.manor" - do presenter test setup - go next - - # A1, B* - frame test_presence_available - do test presence available - do salt raet presenter - do test presence available check - do presenter test cleanup - go next - - # A2 - frame test_presence_joined - do test presence joined - do salt raet presenter - do test presence joined check - do presenter test cleanup - go next - - # A3 - frame test_presence_allowed - do test presence allowed - do salt raet presenter - do test presence allowed check - do presenter test cleanup - go next - - # A4 - frame test_presence_alived - do test presence alived - do salt raet presenter - do test presence alived check - do presenter test cleanup - go next - - # A5 - frame test_presence_reaped - do test presence reaped - do salt raet presenter - do test presence reaped check - do presenter test cleanup - go next - - # C1 - frame test_presence_no_request - do test presence no request - do salt raet presenter - do test presence no request check - do presenter test cleanup - go next - - # C2 - frame test_presence_unknown_src - do test presence unknown src - do salt raet presenter - do test presence unknown src check - do presenter test cleanup - go next - - # D1 - frame test_presence_available_no_minions - do test presence available no minions - do salt raet presenter - do test presence available no minions check - do presenter test cleanup - go next - - # D2 - frame test_presence_available_one_minion - do test presence available one minion - do salt raet presenter - do test presence available one minion check - do presenter test cleanup - go next - - # D3 - frame test_presence_available_unknown_ip - do test presence available unknown ip - do salt raet presenter - do test presence available unknown ip check - do presenter test cleanup - go next - - # D4 - frame test_presence_allowed_no_minions - do test presence allowed no minions - do salt raet presenter - do test presence allowed no minions - do presenter test cleanup - go next - - # D4 - frame test_presence_allowed_one_minion - do test presence allowed one minion - do salt raet presenter - do test presence allowed one minion - do presenter test cleanup - go next - - frame done - go abort if meta.failure - go success - - frame success - print "Success" - bid stop all - - frame abort - print "Failure" - bid stop all diff --git a/salt/daemons/test/test_master.py b/salt/daemons/test/test_master.py deleted file mode 100644 index d99b9e30008..00000000000 --- a/salt/daemons/test/test_master.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -''' -Runs minion floscript - -''' -from __future__ import absolute_import, print_function, unicode_literals -# pylint: skip-file -import os -import stat - -from ioflo.aid.odicting import odict -from ioflo.base.consoling import getConsole -console = getConsole() - -import salt.daemons.flo - -def test(): - """ Execute run.start """ - - pkiDirpath = os.path.join('/tmp', 'raet', 'testo', 'master', 'pki') - if not os.path.exists(pkiDirpath): - os.makedirs(pkiDirpath) - - keyDirpath = os.path.join('/tmp', 'raet', 'testo', 'key') - if not os.path.exists(keyDirpath): - os.makedirs(keyDirpath) - - acceptedDirpath = os.path.join(pkiDirpath, 'accepted') - if not os.path.exists(acceptedDirpath): - os.makedirs(acceptedDirpath) - - pendingDirpath = os.path.join(pkiDirpath, 'pending') - if not os.path.exists(pendingDirpath): - os.makedirs(pendingDirpath) - - rejectedDirpath = os.path.join(pkiDirpath, 'rejected') - if not os.path.exists(rejectedDirpath): - os.makedirs(rejectedDirpath) - - localFilepath = os.path.join(pkiDirpath, 'local.key') - if os.path.exists(localFilepath): - mode = os.stat(localFilepath).st_mode - print(mode) - os.chmod(localFilepath, mode | stat.S_IWUSR | stat.S_IRUSR) - mode = os.stat(localFilepath).st_mode - print(mode) - - cacheDirpath = os.path.join('/tmp/raet', 'cache', 'master') - if not os.path.exists(cacheDirpath): - os.makedirs(cacheDirpath) - - sockDirpath = os.path.join('/tmp/raet', 'sock', 'master') - if not os.path.exists(sockDirpath): - os.makedirs(sockDirpath) - - filepath = 'master.flo' - opts = dict( - id='master', - __role='master', - ioflo_period=0.1, - ioflo_realtime=True, - master_floscript=filepath, - ioflo_verbose=2, - interface="", - raet_port=7530, - transport='raet', - client_acl=dict(), - publisher_acl=dict(), - pki_dir=pkiDirpath, - key_dir=keyDirpath, - sock_dir=sockDirpath, - cachedir=cacheDirpath, - open_mode=True, - auto_accept=True, - client_acl_verify=True, - ) - - master = salt.daemons.flo.IofloMaster(opts=opts) - master.start(behaviors=['raet.flo.behaving']) - -if __name__ == '__main__': - console.reinit(verbosity=console.Wordage.concise) - test() diff --git a/salt/daemons/test/test_minion.py b/salt/daemons/test/test_minion.py deleted file mode 100644 index dbb48282c95..00000000000 --- a/salt/daemons/test/test_minion.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -''' -Runs minion floscript -''' -from __future__ import absolute_import, print_function, unicode_literals - -import os -import stat - -from ioflo.base.consoling import getConsole -console = getConsole() - -import salt.daemons.flo - -FLO_DIR_PATH = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'flo' -) - - -def test(): - """ Execute run.start """ - - pkiDirpath = os.path.join('/tmp', 'raet', 'testo', 'minion', 'pki') - if not os.path.exists(pkiDirpath): - os.makedirs(pkiDirpath) - - acceptedDirpath = os.path.join(pkiDirpath, 'accepted') - if not os.path.exists(acceptedDirpath): - os.makedirs(acceptedDirpath) - - pendingDirpath = os.path.join(pkiDirpath, 'pending') - if not os.path.exists(pendingDirpath): - os.makedirs(pendingDirpath) - - rejectedDirpath = os.path.join(pkiDirpath, 'rejected') - if not os.path.exists(rejectedDirpath): - os.makedirs(rejectedDirpath) - - localFilepath = os.path.join(pkiDirpath, 'local.key') - if os.path.exists(localFilepath): - mode = os.stat(localFilepath).st_mode - print(mode) - os.chmod(localFilepath, mode | stat.S_IWUSR | stat.S_IRUSR) - mode = os.stat(localFilepath).st_mode - print(mode) - - cacheDirpath = os.path.join('/tmp/raet', 'cache', 'minion') - if not os.path.exists(cacheDirpath): - os.makedirs(cacheDirpath) - - sockDirpath = os.path.join('/tmp/raet', 'sock', 'minion') - if not os.path.exists(sockDirpath): - os.makedirs(sockDirpath) - - #filepath = os.path.join(FLO_DIR_PATH, 'minion.flo') - filepath = 'minion.flo' - opts = dict( - id="minion", - __role='minion', - ioflo_period=0.1, - ioflo_realtime=True, - minion_floscript=filepath, - ioflo_verbose=2, - interface="", - raet_port=7531, - master_port=7530, - master='127.0.0.1', - transport='raet', - client_acl=dict(), - publisher_acl=dict(), - pki_dir=pkiDirpath, - sock_dir=sockDirpath, - cachedir=cacheDirpath, - open_mode=True, - auto_accept=True) - - minion = salt.daemons.flo.IofloMinion(opts=opts) - minion.start(behaviors=['raet.flo.behaving']) - - -if __name__ == '__main__': - console.reinit(verbosity=console.Wordage.concise) - test() diff --git a/salt/daemons/test/test_multimaster.py b/salt/daemons/test/test_multimaster.py deleted file mode 100644 index 88114a9d4cb..00000000000 --- a/salt/daemons/test/test_multimaster.py +++ /dev/null @@ -1,355 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Tests of utilities that support multiple masters in Salt Raet - -''' -from __future__ import absolute_import, print_function, unicode_literals -import sys -from salt.ext.six.moves import map -# pylint: disable=blacklisted-import -if sys.version_info < (2, 7): - import unittest2 as unittest -else: - import unittest -# pylint: enable=blacklisted-import - -from ioflo.aid.timing import StoreTimer -from ioflo.base import storing -from ioflo.base.consoling import getConsole -console = getConsole() - -from salt.daemons import parse_hostname, extract_masters - - -def setUpModule(): - console.reinit(verbosity=console.Wordage.concise) - - -def tearDownModule(): - pass - - -class BasicTestCase(unittest.TestCase): # pylint: disable=moved-test-case-class - - def setUp(self): - self.store = storing.Store(stamp=0.0) - self.timer = StoreTimer(store=self.store, duration=1.0) - self.port = 4506 - self.opts = dict(master_port=self.port) - - def tearDown(self): - pass - - def testParseHostname(self): - ''' - Test parsing hostname provided according to syntax for opts['master'] - ''' - console.terse("{0}\n".format(self.testParseHostname.__doc__)) - - self.assertEquals(parse_hostname('localhost', self.port), - ('localhost', 4506)) - self.assertEquals(parse_hostname('127.0.0.1', self.port), - ('127.0.0.1', 4506)) - self.assertEquals(parse_hostname('10.0.2.100', self.port), - ('10.0.2.100', 4506)) - self.assertEquals(parse_hostname('me.example.com', self.port), - ('me.example.com', 4506)) - self.assertEquals(parse_hostname( - '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa', - self.port), - ('1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa', - 4506)) - self.assertEquals(parse_hostname('fe80::1%lo0', self.port), - ('fe80::1%lo0', 4506)) - - self.assertEquals(parse_hostname(' localhost ', self.port), - ('localhost', 4506)) - self.assertEquals(parse_hostname(' 127.0.0.1 ', self.port), - ('127.0.0.1', 4506)) - self.assertEquals(parse_hostname(' 10.0.2.100 ', self.port), - ('10.0.2.100', 4506)) - self.assertEquals(parse_hostname(' me.example.com ', self.port), - ('me.example.com', 4506)) - self.assertEquals(parse_hostname( - ' 1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa ', - self.port), - ('1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa', - 4506)) - self.assertEquals(parse_hostname(' fe80::1%lo0 ', self.port), - ('fe80::1%lo0', 4506)) - - self.assertEquals(parse_hostname('localhost 4510', self.port), - ('localhost', 4510)) - self.assertEquals(parse_hostname('127.0.0.1 4510', self.port), - ('127.0.0.1', 4510)) - self.assertEquals(parse_hostname('10.0.2.100 4510', self.port), - ('10.0.2.100', 4510)) - self.assertEquals(parse_hostname('me.example.com 4510', self.port), - ('me.example.com', 4510)) - self.assertEquals(parse_hostname( - '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa 4510', - self.port), - ('1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa', - 4510)) - self.assertEquals(parse_hostname('fe80::1%lo0 4510', self.port), - ('fe80::1%lo0', 4510)) - - self.assertEquals(parse_hostname(' localhost 4510 ', self.port), - ('localhost', 4510)) - self.assertEquals(parse_hostname(' 127.0.0.1 4510 ', self.port), - ('127.0.0.1', 4510)) - self.assertEquals(parse_hostname(' 10.0.2.100 4510 ', self.port), - ('10.0.2.100', 4510)) - self.assertEquals(parse_hostname(' me.example.com 4510 ', self.port), - ('me.example.com', 4510)) - self.assertEquals(parse_hostname( - ' 1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa 4510 ', - self.port), - ('1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa', - 4510)) - self.assertEquals(parse_hostname(' fe80::1%lo0 4510 ', self.port), - ('fe80::1%lo0', 4510)) - - self.assertEquals(parse_hostname('localhost abcde', self.port), None) - self.assertEquals(parse_hostname('127.0.0.1 a4510', self.port), None) - self.assertEquals(parse_hostname(list([1, 2, 3]), self.port), None) - self.assertEquals(parse_hostname(list(), self.port), None) - self.assertEquals(parse_hostname(dict(a=1), self.port), None) - self.assertEquals(parse_hostname(dict(), self.port), None) - self.assertEquals(parse_hostname(4510, self.port), None) - self.assertEquals(parse_hostname(('localhost', 4510), self.port), None) - - self.assertEquals(parse_hostname('localhost:4510', self.port), - ('localhost', 4510)) - self.assertEquals(parse_hostname('127.0.0.1:4510', self.port), - ('127.0.0.1', 4510)) - self.assertEquals(parse_hostname('10.0.2.100:4510', self.port), - ('10.0.2.100', 4510)) - self.assertEquals(parse_hostname('me.example.com:4510', self.port), - ('me.example.com', 4510)) - self.assertEquals(parse_hostname( - '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa:4510', - self.port), - ('1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa', - 4510)) - self.assertEquals(parse_hostname('fe80::1%lo0:4510', self.port), - ('fe80::1%lo0:4510', 4506)) - self.assertEquals(parse_hostname('localhost::4510', self.port), - ('localhost::4510', 4506)) - - def testExtractMastersSingle(self): - ''' - Test extracting from master provided according to syntax for opts['master'] - ''' - console.terse("{0}\n".format(self.testExtractMastersSingle.__doc__)) - - master = 'localhost' - self.opts.update(master=master) - self.assertEquals(extract_masters(self.opts), - [ - dict(external=('localhost', 4506), - internal=None), - ]) - - master = '127.0.0.1' - self.opts.update(master=master) - self.assertEquals(extract_masters(self.opts), - [ - dict(external=('127.0.0.1', 4506), - internal=None), - ]) - - master = 'localhost 4510' - self.opts.update(master=master) - self.assertEquals(extract_masters(self.opts), - [ - dict(external=('localhost', 4510), - internal=None), - ]) - - master = '127.0.0.1 4510' - self.opts.update(master=master) - self.assertEquals(extract_masters(self.opts), - [ - dict(external=('127.0.0.1', 4510), - internal=None), - ]) - - master = '10.0.2.23' - self.opts.update(master=master) - self.assertEquals(extract_masters(self.opts), - [ - dict(external=('10.0.2.23', 4506), - internal=None), - ]) - - master = 'me.example.com' - self.opts.update(master=master) - self.assertEquals(extract_masters(self.opts), - [ - dict(external=('me.example.com', 4506), - internal=None), - ]) - - master = '10.0.2.23 4510' - self.opts.update(master=master) - self.assertEquals(extract_masters(self.opts), - [ - dict(external=('10.0.2.23', 4510), - internal=None), - ]) - - master = 'me.example.com 4510' - self.opts.update(master=master) - self.assertEquals(extract_masters(self.opts), - [ - dict(external=('me.example.com', 4510), - internal=None), - ]) - - master = dict(external='10.0.2.23 4510') - self.opts.update(master=master) - self.assertEquals(extract_masters(self.opts), - [ - dict(external=('10.0.2.23', 4510), - internal=None), - ]) - - master = dict(external='10.0.2.23 4510', internal='') - self.opts.update(master=master) - self.assertEquals(extract_masters(self.opts), - [ - dict(external=('10.0.2.23', 4510), - internal=None), - ]) - - master = dict(internal='10.0.2.23 4510') - self.opts.update(master=master) - self.assertEquals(extract_masters(self.opts), []) - - def testExtractMastersMultiple(self): - ''' - Test extracting from master provided according to syntax for opts['master'] - ''' - console.terse("{0}\n".format(self.testExtractMastersMultiple.__doc__)) - - master = [ - 'localhost', - '10.0.2.23', - 'me.example.com' - ] - self.opts.update(master=master) - self.assertEquals(extract_masters(self.opts), - [ - { - 'external': ('localhost', 4506), - 'internal': None - }, - { - 'external': ('10.0.2.23', 4506), - 'internal': None - }, - { - 'external': ('me.example.com', 4506), - 'internal': None - }, - ]) - - master = [ - 'localhost 4510', - '10.0.2.23 4510', - 'me.example.com 4510' - ] - self.opts.update(master=master) - self.assertEquals(extract_masters(self.opts), - [ - { - 'external': ('localhost', 4510), - 'internal': None - }, - { - 'external': ('10.0.2.23', 4510), - 'internal': None - }, - { - 'external': ('me.example.com', 4510), - 'internal': None - }, - ]) - - master = [ - { - 'external': 'localhost 4510', - 'internal': '', - }, - { - 'external': 'me.example.com 4510', - 'internal': '10.0.2.23 4510', - }, - { - 'external': 'you.example.com 4509', - } - ] - self.opts.update(master=master) - self.assertEquals(extract_masters(self.opts), - [ - { - 'external': ('localhost', 4510), - 'internal': None - }, - { - 'external': ('me.example.com', 4510), - 'internal': ('10.0.2.23', 4510) - }, - { - 'external': ('you.example.com', 4509), - 'internal': None - }, - ]) - - -def runOne(test): - ''' - Unittest Runner - ''' - test = BasicTestCase(test) - suite = unittest.TestSuite([test]) - unittest.TextTestRunner(verbosity=2).run(suite) - - -def runSome(): - ''' - Unittest runner - ''' - tests = [] - names = [ - 'testParseHostname', - 'testExtractMastersSingle', - 'testExtractMastersMultiple', - ] - - tests.extend(list(list(map(BasicTestCase, names)))) - - suite = unittest.TestSuite(tests) - unittest.TextTestRunner(verbosity=2).run(suite) - - -def runAll(): - ''' - Unittest runner - ''' - suite = unittest.TestSuite() - suite.addTest(unittest.TestLoader().loadTestsFromTestCase(BasicTestCase)) - - unittest.TextTestRunner(verbosity=2).run(suite) - - -if __name__ == '__main__' and __package__ is None: - - #console.reinit(verbosity=console.Wordage.concise) - - runAll() # run all unittests - - #runSome() # only run some - - #runOne('testParseHostname') diff --git a/salt/daemons/test/test_plans.py b/salt/daemons/test/test_plans.py deleted file mode 100644 index 98e16f7cb83..00000000000 --- a/salt/daemons/test/test_plans.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -''' -Runs all the example FloScripts - - -''' -# pylint: disable=3rd-party-module-not-gated - -# Import Python Libs -from __future__ import absolute_import, print_function, unicode_literals -import os - -# Import 3rd-party libs -import ioflo.app.run -from ioflo.base.consoling import getConsole - -console = getConsole() - -PLAN_DIR_PATH = os.path.join( - os.path.dirname(os.path.abspath(__file__)), 'plan' -) - - -def getPlanFiles(planDirPath=PLAN_DIR_PATH): - planFiles = [] - for fname in os.listdir(os.path.abspath(planDirPath)): - root, ext = os.path.splitext(fname) - if ext != '.flo' or root.startswith('__'): - continue - - planFiles.append(os.path.abspath(os.path.join(planDirPath, fname))) - return planFiles - - -def main(): - ''' - Run example scripts - ''' - console.concise('Test started') - behaviors = ['salt.daemons.flo', 'salt.daemons.test.plan'] - failedCount = 0 - plans = getPlanFiles() - for plan in plans: - name, ext = os.path.splitext(os.path.basename(plan)) - skeddar = ioflo.app.run.run(name=name, - filepath=plan, - behaviors=behaviors, - period=0.0625, - verbose=1, - real=False,) - - print('Plan {0}\n Skeddar {1}\n'.format(plan, skeddar.name)) - failed = False - for house in skeddar.houses: - failure = house.metas['failure'].value - if failure: - failed = True - print('**** Failed in House = {0}. ' - 'Failure = {1}.\n'.format(house.name, failure)) - else: - print('**** Succeeded in House = {0}.\n'.format(house.name)) - if failed: - failedCount += 1 - - print('{0} failed out of {1}.\n'.format(failedCount, len(plans))) - - -if __name__ == '__main__': - console.reinit(verbosity=console.Wordage.profuse) - main() diff --git a/salt/daemons/test/test_presence.py b/salt/daemons/test/test_presence.py deleted file mode 100644 index e98e8eb698b..00000000000 --- a/salt/daemons/test/test_presence.py +++ /dev/null @@ -1,740 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Raet Ioflo Behavior Unittests -''' -from __future__ import absolute_import, print_function, unicode_literals -import sys -from salt.ext.six.moves import map -import importlib -# pylint: disable=blacklisted-import -if sys.version_info < (2, 7): - import unittest2 as unittest -else: - import unittest -# pylint: enable=blacklisted-import - -from ioflo.base.consoling import getConsole -console = getConsole() -from ioflo.aid.odicting import odict -from ioflo.test import testing - -from raet.lane.stacking import LaneStack -from raet.stacking import Stack - -import salt.utils.stringutils -from salt.utils.event import tagify - - -def setUpModule(): - console.reinit(verbosity=console.Wordage.concise) - - -def tearDownModule(): - pass - - -class PresenterTestCase(testing.FrameIofloTestCase): - ''' - Test case for Salt Raet Presenter deed - ''' - def setUp(self): - ''' - Call super if override so House Framer and Frame are setup correctly - ''' - behaviors = ['salt.daemons.flo', 'salt.daemons.test.plan'] - for behavior in behaviors: - mod = importlib.import_module(behavior) - super(PresenterTestCase, self).setUp() - - def tearDown(self): - ''' - Call super if override so House Framer and Frame are torn down correctly - ''' - super(PresenterTestCase, self).tearDown() - - def addPresenceInfo(self, stateGrp, name, ip, port): - self.assertIn(stateGrp, ('alloweds', 'aliveds', 'reapeds')) - group = self.store.fetch('.salt.var.presence.{0}'.format(stateGrp)) - if group.value is None: - group.value = odict() - remote = Stack() - remote.ha = (ip, port) - group.value[name] = remote - - def addAvailable(self, name): - availables = self.store.fetch('.salt.var.presence.availables') - if availables.value is None: - availables.value = set() - availables.value.add(name) - - def testContextSetup(self): - ''' - Test the context setup procedure used in all the consequence tests works as expected - This test intended to avoid some checks in other tests - ''' - console.terse("{0}\n".format(self.testContextSetup.__doc__)) - - act = self.addEnterDeed("TestOptsSetupMaster") - self.assertIn(act, self.frame.enacts) - self.assertEqual(act.actor, "TestOptsSetupMaster") - act = self.addEnterDeed("SaltRaetManorLaneSetup") - self.assertIn(act, self.frame.enacts) - self.assertEqual(act.actor, "SaltRaetManorLaneSetup") - act = self.addEnterDeed("PresenterTestSetup") - self.assertIn(act, self.frame.enacts) - self.assertEqual(act.actor, "PresenterTestSetup") - - act = self.addRecurDeed("SaltRaetPresenter") - self.assertIn(act, self.frame.reacts) - self.assertEqual(act.actor, "SaltRaetPresenter") - - self.resolve() # resolve House, Framer, Frame, Acts, Actors - - self.frame.enter() - self.assertDictEqual( - act.actor.Ioinits, - {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'), - 'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'), - 'alloweds': salt.utils.stringutils.to_str('.salt.var.presence.alloweds'), - 'aliveds': salt.utils.stringutils.to_str('.salt.var.presence.aliveds'), - 'reapeds': salt.utils.stringutils.to_str('.salt.var.presence.reapeds'), - 'availables': salt.utils.stringutils.to_str('.salt.var.presence.availables')}) - - self.assertTrue(hasattr(act.actor, 'opts')) - self.assertTrue(hasattr(act.actor, 'presence_req')) - self.assertTrue(hasattr(act.actor, 'lane_stack')) - self.assertTrue(hasattr(act.actor, 'alloweds')) - self.assertTrue(hasattr(act.actor, 'aliveds')) - self.assertTrue(hasattr(act.actor, 'reapeds')) - self.assertTrue(hasattr(act.actor, 'availables')) - self.assertIsInstance(act.actor.lane_stack.value, LaneStack) - self.frame.recur() - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - def testPresenceAvailable(self): - ''' - Test Presenter 'available' request (A1, B*) - ''' - console.terse("{0}\n".format(self.testPresenceAvailable.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("PresenterTestSetup") - act = self.addRecurDeed("SaltRaetPresenter") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add available minions - self.addAvailable('alpha') - self.addAvailable('beta') - self.addPresenceInfo('aliveds', 'alpha', '1.1.1.1', '1234') - self.addPresenceInfo('aliveds', 'beta', '1.2.3.4', '1234') - # add presence request - testStack = self.store.fetch('.salt.test.lane.stack').value - presenceReq = self.store.fetch('.salt.presence.event_req').value - ryn = 'manor' - # general available request format - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'available'}}) - # missing 'data', fallback to available - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}}) - # missing 'state' in 'data', fallback to available - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {}}) - # requested None state, fallback to available - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': None}}) - # requested 'present' state that is alias for available - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'present'}}) - - # Test - # process 5 requests at once - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 5) - - tag = tagify('present', 'presence') - while testStack.rxMsgs: - msg, sender = testStack.rxMsgs.popleft() - self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'present': {'alpha': '1.1.1.1', - 'beta': '1.2.3.4'}}}) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - def testPresenceJoined(self): - ''' - Test Presenter 'joined' request (A2) - ''' - console.terse("{0}\n".format(self.testPresenceJoined.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("PresenterTestSetup") - act = self.addRecurDeed("SaltRaetPresenter") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add joined minions - # NOTE: for now alloweds are threaded as joineds - self.addPresenceInfo('alloweds', 'alpha', '1.1.1.1', '1234') - self.addPresenceInfo('alloweds', 'beta', '1.2.3.4', '1234') - # add presence request - testStack = self.store.fetch('.salt.test.lane.stack').value - presenceReq = self.store.fetch('.salt.presence.event_req').value - ryn = 'manor' - msg = {'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'joined'}} - presenceReq.append(msg) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 1) - - tag = tagify('present', 'presence') - msg, sender = testStack.rxMsgs.popleft() - self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'joined': {'alpha': '1.1.1.1', - 'beta': '1.2.3.4'}}}) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - def testPresenceAllowed(self): - ''' - Test Presenter 'allowed' request (A3) - ''' - console.terse("{0}\n".format(self.testPresenceAllowed.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("PresenterTestSetup") - act = self.addRecurDeed("SaltRaetPresenter") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add allowed minions - self.addPresenceInfo('alloweds', 'alpha', '1.1.1.1', '1234') - self.addPresenceInfo('alloweds', 'beta', '1.2.3.4', '1234') - # add presence request - testStack = self.store.fetch('.salt.test.lane.stack').value - presenceReq = self.store.fetch('.salt.presence.event_req').value - ryn = 'manor' - msg = {'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'allowed'}} - presenceReq.append(msg) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 1) - - tag = tagify('present', 'presence') - msg, sender = testStack.rxMsgs.popleft() - self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'allowed': {'alpha': '1.1.1.1', - 'beta': '1.2.3.4'}}}) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - def testPresenceAlived(self): - ''' - Test Presenter 'alived' request (A4) - ''' - console.terse("{0}\n".format(self.testPresenceAlived.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("PresenterTestSetup") - act = self.addRecurDeed("SaltRaetPresenter") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add alived minions - self.addPresenceInfo('aliveds', 'alpha', '1.1.1.1', '1234') - self.addPresenceInfo('aliveds', 'beta', '1.2.3.4', '1234') - # add presence request - testStack = self.store.fetch('.salt.test.lane.stack').value - presenceReq = self.store.fetch('.salt.presence.event_req').value - ryn = 'manor' - msg = {'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'alived'}} - presenceReq.append(msg) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 1) - - tag = tagify('present', 'presence') - msg, sender = testStack.rxMsgs.popleft() - self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'alived': {'alpha': '1.1.1.1', - 'beta': '1.2.3.4'}}}) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - def testPresenceReaped(self): - ''' - Test Presenter 'reaped' request (A5) - ''' - console.terse("{0}\n".format(self.testPresenceReaped.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("PresenterTestSetup") - act = self.addRecurDeed("SaltRaetPresenter") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add reaped minions - self.addPresenceInfo('reapeds', 'alpha', '1.1.1.1', '1234') - self.addPresenceInfo('reapeds', 'beta', '1.2.3.4', '1234') - # add presence request - testStack = self.store.fetch('.salt.test.lane.stack').value - presenceReq = self.store.fetch('.salt.presence.event_req').value - ryn = 'manor' - msg = {'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'reaped'}} - presenceReq.append(msg) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 1) - - tag = tagify('present', 'presence') - msg, sender = testStack.rxMsgs.popleft() - self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'reaped': {'alpha': '1.1.1.1', - 'beta': '1.2.3.4'}}}) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - def testPresenceNoRequest(self): - ''' - Test Presenter with no requests (C1) - ''' - console.terse("{0}\n".format(self.testPresenceNoRequest.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("PresenterTestSetup") - act = self.addRecurDeed("SaltRaetPresenter") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Test - self.frame.recur() # run in frame - - # Check - testStack = self.store.fetch('.salt.test.lane.stack').value - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 0) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - def testPresenceUnknownSrc(self): - ''' - Test Presenter handles request from unknown (disconnected) source (C2) - ''' - console.terse("{0}\n".format(self.testPresenceUnknownSrc.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("PresenterTestSetup") - act = self.addRecurDeed("SaltRaetPresenter") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add presence request - testStack = self.store.fetch('.salt.test.lane.stack').value - presenceReq = self.store.fetch('.salt.presence.event_req').value - ryn = 'manor' - name = 'unknown_name' - self.assertNotEqual(name, testStack.local.name) - msg = {'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, name, None)}} - presenceReq.append(msg) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 0) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - def testPresenceAvailableNoMinions(self): - ''' - Test Presenter 'available' request with no minions in the state (D1) - ''' - console.terse("{0}\n".format(self.testPresenceAvailableNoMinions.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("PresenterTestSetup") - act = self.addRecurDeed("SaltRaetPresenter") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add presence request - testStack = self.store.fetch('.salt.test.lane.stack').value - presenceReq = self.store.fetch('.salt.presence.event_req').value - ryn = 'manor' - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'available'}}) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 1) - - tag = tagify('present', 'presence') - msg, sender = testStack.rxMsgs.popleft() - self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'present': {}}}) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - def testPresenceAvailableOneMinion(self): - ''' - Test Presenter 'available' request with one minion in the state (D2) - ''' - console.terse("{0}\n".format(self.testPresenceAvailableOneMinion.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("PresenterTestSetup") - act = self.addRecurDeed("SaltRaetPresenter") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add available minions - self.addAvailable('alpha') - self.addPresenceInfo('aliveds', 'alpha', '1.1.1.1', '1234') - # add presence request - testStack = self.store.fetch('.salt.test.lane.stack').value - presenceReq = self.store.fetch('.salt.presence.event_req').value - ryn = 'manor' - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'available'}}) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 1) - - tag = tagify('present', 'presence') - msg, sender = testStack.rxMsgs.popleft() - self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'present': {'alpha': '1.1.1.1'}}}) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - def testPresenceAvailableSomeIpUnknown(self): - ''' - Test Presenter 'available' request with some minion addresses aren't known (D3) - ''' - console.terse("{0}\n".format(self.testPresenceAvailableSomeIpUnknown.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("PresenterTestSetup") - act = self.addRecurDeed("SaltRaetPresenter") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add available minions - self.addAvailable('alpha') - self.addAvailable('beta') - self.addAvailable('gamma') - self.addPresenceInfo('aliveds', 'alpha', '1.1.1.1', '1234') - self.addPresenceInfo('aliveds', 'delta', '1.2.3.4', '1234') - # add presence request - testStack = self.store.fetch('.salt.test.lane.stack').value - presenceReq = self.store.fetch('.salt.presence.event_req').value - ryn = 'manor' - presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'available'}}) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 1) - - tag = tagify('present', 'presence') - msg, sender = testStack.rxMsgs.popleft() - self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'present': {'alpha': '1.1.1.1', - 'beta': None, - 'gamma': None}}}) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - def testPresenceAllowedNoMinions(self): - ''' - Test Presenter 'allowed' request with no minions in the state (D4) - ''' - console.terse("{0}\n".format(self.testPresenceAllowedNoMinions.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("PresenterTestSetup") - act = self.addRecurDeed("SaltRaetPresenter") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add presence request - testStack = self.store.fetch('.salt.test.lane.stack').value - presenceReq = self.store.fetch('.salt.presence.event_req').value - ryn = 'manor' - msg = {'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'allowed'}} - presenceReq.append(msg) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 1) - - tag = tagify('present', 'presence') - msg, sender = testStack.rxMsgs.popleft() - self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'allowed': {}}}) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - def testPresenceAllowedOneMinion(self): - ''' - Test Presenter 'allowed' request with one minion in the state (D5) - ''' - console.terse("{0}\n".format(self.testPresenceAllowedOneMinion.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("PresenterTestSetup") - act = self.addRecurDeed("SaltRaetPresenter") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add allowed minions - self.addPresenceInfo('alloweds', 'alpha', '1.1.1.1', '1234') - # add presence request - testStack = self.store.fetch('.salt.test.lane.stack').value - presenceReq = self.store.fetch('.salt.presence.event_req').value - ryn = 'manor' - msg = {'route': {'dst': (None, ryn, 'presence_req'), - 'src': (None, testStack.local.name, None)}, - 'data': {'state': 'allowed'}} - presenceReq.append(msg) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 1) - - tag = tagify('present', 'presence') - msg, sender = testStack.rxMsgs.popleft() - self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'allowed': {'alpha': '1.1.1.1'}}}) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - -def runOne(test): - ''' - Unittest Runner - ''' - test = PresenterTestCase(test) - suite = unittest.TestSuite([test]) - unittest.TextTestRunner(verbosity=2).run(suite) - - -def runSome(): - ''' - Unittest runner - ''' - tests = [] - names = [ - 'testContextSetup', - 'testPresenceAvailable', - 'testPresenceJoined', - 'testPresenceAllowed', - 'testPresenceAlived', - 'testPresenceReaped', - 'testPresenceNoRequest', - 'testPresenceUnknownSrc', - 'testPresenceAvailableNoMinions', - 'testPresenceAvailableOneMinion', - 'testPresenceAvailableSomeIpUnknown', - 'testPresenceAllowedNoMinions', - 'testPresenceAllowedOneMinion', - ] - tests.extend(list(map(PresenterTestCase, names))) - suite = unittest.TestSuite(tests) - unittest.TextTestRunner(verbosity=2).run(suite) - - -def runAll(): - ''' - Unittest runner - ''' - suite = unittest.TestSuite() - suite.addTest(unittest.TestLoader().loadTestsFromTestCase(PresenterTestCase)) - unittest.TextTestRunner(verbosity=2).run(suite) - - -if __name__ == '__main__' and __package__ is None: - - # console.reinit(verbosity=console.Wordage.concise) - - runAll() # run all unittests - - # runSome() #only run some - - # runOne('testPresenceAvailable') diff --git a/salt/daemons/test/test_raetkey.py b/salt/daemons/test/test_raetkey.py deleted file mode 100644 index b4a02188810..00000000000 --- a/salt/daemons/test/test_raetkey.py +++ /dev/null @@ -1,442 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Tests to try out salt key.RaetKey Potentially ephemeral - -''' -from __future__ import absolute_import, print_function, unicode_literals -# pylint: skip-file -# pylint: disable=C0103 -import sys -import salt.utils.stringutils -from salt.ext.six.moves import map -if sys.version_info < (2, 7): - import unittest2 as unittest -else: - import unittest - -import os -import stat -import time -import tempfile -import shutil - -from ioflo.aid.odicting import odict -from ioflo.aid.timing import Timer, StoreTimer -from ioflo.base import storing -from ioflo.base.consoling import getConsole -console = getConsole() - -from raet import raeting, nacling -from raet.road import estating, keeping, stacking - -from salt.key import RaetKey - - -def setUpModule(): - console.reinit(verbosity=console.Wordage.concise) - - -def tearDownModule(): - pass - - -class BasicTestCase(unittest.TestCase): - """""" - - def setUp(self): - self.store = storing.Store(stamp=0.0) - self.timer = StoreTimer(store=self.store, duration=1.0) - - self.saltDirpath = tempfile.mkdtemp(prefix="salt", suffix="main", dir='/tmp') - - pkiDirpath = os.path.join(self.saltDirpath, 'pki') - if not os.path.exists(pkiDirpath): - os.makedirs(pkiDirpath) - - acceptedDirpath = os.path.join(pkiDirpath, 'accepted') - if not os.path.exists(acceptedDirpath): - os.makedirs(acceptedDirpath) - - pendingDirpath = os.path.join(pkiDirpath, 'pending') - if not os.path.exists(pendingDirpath): - os.makedirs(pendingDirpath) - - rejectedDirpath = os.path.join(pkiDirpath, 'rejected') - if not os.path.exists(rejectedDirpath): - os.makedirs(rejectedDirpath) - - self.localFilepath = os.path.join(pkiDirpath, 'local.key') - if os.path.exists(self.localFilepath): - mode = os.stat(self.localFilepath).st_mode - os.chmod(self.localFilepath, mode | stat.S_IWUSR | stat.S_IWUSR) - - self.cacheDirpath = os.path.join(self.saltDirpath, 'cache') - self.sockDirpath = os.path.join(self.saltDirpath, 'sock') - - self.opts = dict( - __role='master', - id='master', - pki_dir=pkiDirpath, - sock_dir=self.sockDirpath, - cachedir=self.cacheDirpath, - open_mode=False, - auto_accept=True, - transport='raet', - ) - - self.mainKeeper = RaetKey(opts=self.opts) - self.baseDirpath = tempfile.mkdtemp(prefix="salt", suffix="base", dir='/tmp') - - def tearDown(self): - if os.path.exists(self.saltDirpath): - shutil.rmtree(self.saltDirpath) - - def createRoadData(self, name, base): - ''' - Creates odict and populates with data to setup road stack - { - name: stack name local estate name - dirpath: dirpath for keep files - sighex: signing key - verhex: verify key - prihex: private key - pubhex: public key - } - ''' - data = odict() - data['name'] = name - data['dirpath'] = os.path.join(base, 'road', 'keep', name) - signer = nacling.Signer() - data['sighex'] = signer.keyhex - data['verhex'] = signer.verhex - privateer = nacling.Privateer() - data['prihex'] = privateer.keyhex - data['pubhex'] = privateer.pubhex - - return data - - def testAutoAccept(self): - ''' - Basic function of RaetKey in auto accept mode - ''' - console.terse("{0}\n".format(self.testAutoAccept.__doc__)) - self.opts['auto_accept'] = True - self.assertTrue(self.opts['auto_accept']) - self.assertDictEqual(self.mainKeeper.all_keys(), {'accepted': [], - 'local': [], - 'rejected': [], - 'pending': []}) - - localkeys = self.mainKeeper.read_local() - self.assertDictEqual(localkeys, {}) - - main = self.createRoadData(name='main', base=self.baseDirpath) - self.mainKeeper.write_local(main['prihex'], main['sighex']) - localkeys = self.mainKeeper.read_local() - self.assertDictEqual(localkeys, - {'priv': salt.utils.stringutils.to_str(main['prihex']), - 'sign': salt.utils.stringutils.to_str(main['sighex'])}) - allkeys = self.mainKeeper.all_keys() - self.assertDictEqual(allkeys, {'accepted': [], - 'local': [self.localFilepath], - 'rejected': [], - 'pending': []}) - - other1 = self.createRoadData(name='other1', base=self.baseDirpath) - other2 = self.createRoadData(name='other2', base=self.baseDirpath) - - status = self.mainKeeper.status(other1['name'], other1['pubhex'], other1['verhex']) - self.assertEqual(status, 'accepted') - status = self.mainKeeper.status(other2['name'], other2['pubhex'], other2['verhex']) - self.assertEqual(status, 'accepted') - - allkeys = self.mainKeeper.all_keys() - self.assertDictEqual(allkeys, {'accepted': ['other1', 'other2'], - 'local': [self.localFilepath], - 'pending': [], - 'rejected': []}) - - remotekeys = self.mainKeeper.read_remote(other1['name']) - self.assertDictEqual(remotekeys, {'minion_id': 'other1', - 'pub': salt.utils.stringutils.to_str(other1['pubhex']), - 'verify': salt.utils.stringutils.to_str(other1['verhex'])}) - - remotekeys = self.mainKeeper.read_remote(other2['name']) - self.assertDictEqual(remotekeys, {'minion_id': 'other2', - 'pub': salt.utils.stringutils.to_str(other2['pubhex']), - 'verify': salt.utils.stringutils.to_str(other2['verhex'])}) - - listkeys = self.mainKeeper.list_keys() - self.assertDictEqual(listkeys, {'accepted': ['other1', 'other2'], - 'rejected': [], - 'pending': []}) - - allremotekeys = self.mainKeeper.read_all_remote() - self.assertDictEqual(allremotekeys, - {'other1': - {'verify': salt.utils.stringutils.to_str(other1['verhex']), - 'minion_id': 'other1', - 'acceptance': 'accepted', - 'pub': salt.utils.stringutils.to_str(other1['pubhex']), }, - 'other2': - {'verify': salt.utils.stringutils.to_str(other2['verhex']), - 'minion_id': 'other2', - 'acceptance': 'accepted', - 'pub': salt.utils.stringutils.to_str(other2['pubhex']), } - }) - - def testManualAccept(self): - ''' - Basic function of RaetKey in non auto accept mode - ''' - console.terse("{0}\n".format(self.testAutoAccept.__doc__)) - self.opts['auto_accept'] = False - self.assertFalse(self.opts['auto_accept']) - self.assertDictEqual(self.mainKeeper.all_keys(), {'accepted': [], - 'local': [], - 'rejected': [], - 'pending': []}) - - localkeys = self.mainKeeper.read_local() - self.assertDictEqual(localkeys, {}) - - main = self.createRoadData(name='main', base=self.baseDirpath) - self.mainKeeper.write_local(main['prihex'], main['sighex']) - localkeys = self.mainKeeper.read_local() - self.assertDictEqual(localkeys, - {'priv': salt.utils.stringutils.to_str(main['prihex']), - 'sign': salt.utils.stringutils.to_str(main['sighex'])}) - allkeys = self.mainKeeper.all_keys() - self.assertDictEqual(allkeys, {'accepted': [], - 'local': [self.localFilepath], - 'rejected': [], - 'pending': []}) - - other1 = self.createRoadData(name='other1', base=self.baseDirpath) - other2 = self.createRoadData(name='other2', base=self.baseDirpath) - - status = self.mainKeeper.status(other1['name'], other1['pubhex'], other1['verhex']) - self.assertEqual(status, 'pending') - status = self.mainKeeper.status(other2['name'], other2['pubhex'], other2['verhex']) - self.assertEqual(status, 'pending') - - allkeys = self.mainKeeper.all_keys() - self.assertDictEqual(allkeys, {'accepted': [], - 'local': [self.localFilepath], - 'pending': ['other1', 'other2'], - 'rejected': []}) - - remotekeys = self.mainKeeper.read_remote(other1['name']) - self.assertDictEqual(remotekeys, {}) - - remotekeys = self.mainKeeper.read_remote(other2['name']) - self.assertDictEqual(remotekeys, {}) - - listkeys = self.mainKeeper.list_keys() - self.assertDictEqual(listkeys, {'accepted': [], - 'rejected': [], - 'pending': ['other1', 'other2']}) - - allremotekeys = self.mainKeeper.read_all_remote() - self.assertDictEqual(allremotekeys, - {'other1': - {'verify': salt.utils.stringutils.to_str(other1['verhex']), - 'minion_id': 'other1', - 'acceptance': 'pending', - 'pub': salt.utils.stringutils.to_str(other1['pubhex']), - }, - 'other2': - {'verify': salt.utils.stringutils.to_str(other2['verhex']), - 'minion_id': 'other2', - 'acceptance': 'pending', - 'pub': salt.utils.stringutils.to_str(other2['pubhex']), - } - }) - - self.mainKeeper.accept_all() - - allkeys = self.mainKeeper.all_keys() - self.assertDictEqual(allkeys, {'accepted': ['other1', 'other2'], - 'local': [self.localFilepath], - 'pending': [], - 'rejected': []}) - - remotekeys = self.mainKeeper.read_remote(other1['name']) - self.assertDictEqual(remotekeys, {'minion_id': 'other1', - 'pub': salt.utils.stringutils.to_str(other1['pubhex']), - 'verify': salt.utils.stringutils.to_str(other1['verhex'])}) - - remotekeys = self.mainKeeper.read_remote(other2['name']) - self.assertDictEqual(remotekeys, {'minion_id': 'other2', - 'pub': salt.utils.stringutils.to_str(other2['pubhex']), - 'verify': salt.utils.stringutils.to_str(other2['verhex'])}) - - listkeys = self.mainKeeper.list_keys() - self.assertDictEqual(listkeys, {'accepted': ['other1', 'other2'], - 'rejected': [], - 'pending': []}) - - allremotekeys = self.mainKeeper.read_all_remote() - self.assertDictEqual(allremotekeys, - {'other1': - {'verify': salt.utils.stringutils.to_str(other1['verhex']), - 'minion_id': 'other1', - 'acceptance': 'accepted', - 'pub': salt.utils.stringutils.to_str(other1['pubhex']), - }, - 'other2': - {'verify': salt.utils.stringutils.to_str(other2['verhex']), - 'minion_id': 'other2', - 'acceptance': 'accepted', - 'pub': salt.utils.stringutils.to_str(other2['pubhex']), - } - }) - - def testDelete(self): - ''' - Basic function of RaetKey to delete key - ''' - console.terse("{0}\n".format(self.testDelete.__doc__)) - self.opts['auto_accept'] = True - self.assertTrue(self.opts['auto_accept']) - self.assertDictEqual(self.mainKeeper.all_keys(), {'accepted': [], - 'local': [], - 'rejected': [], - 'pending': []}) - - localkeys = self.mainKeeper.read_local() - self.assertDictEqual(localkeys, {}) - - main = self.createRoadData(name='main', base=self.baseDirpath) - self.mainKeeper.write_local(main['prihex'], main['sighex']) - localkeys = self.mainKeeper.read_local() - self.assertDictEqual(localkeys, - {'priv': salt.utils.stringutils.to_str(main['prihex']), - 'sign': salt.utils.stringutils.to_str(main['sighex'])}) - allkeys = self.mainKeeper.all_keys() - self.assertDictEqual(allkeys, {'accepted': [], - 'local': [self.localFilepath], - 'rejected': [], - 'pending': []}) - - other1 = self.createRoadData(name='other1', base=self.baseDirpath) - other2 = self.createRoadData(name='other2', base=self.baseDirpath) - - status = self.mainKeeper.status(other1['name'], other1['pubhex'], other1['verhex']) - self.assertEqual(status, 'accepted') - status = self.mainKeeper.status(other2['name'], other2['pubhex'], other2['verhex']) - self.assertEqual(status, 'accepted') - - allkeys = self.mainKeeper.all_keys() - self.assertDictEqual(allkeys, {'accepted': ['other1', 'other2'], - 'local': [self.localFilepath], - 'pending': [], - 'rejected': []}) - - remotekeys = self.mainKeeper.read_remote(other1['name']) - self.assertDictEqual(remotekeys, {'minion_id': 'other1', - 'pub': salt.utils.stringutils.to_str(other1['pubhex']), - 'verify': salt.utils.stringutils.to_str(other1['verhex']), - }) - - remotekeys = self.mainKeeper.read_remote(other2['name']) - self.assertDictEqual(remotekeys, {'minion_id': 'other2', - 'pub': salt.utils.stringutils.to_str(other2['pubhex']), - 'verify': salt.utils.stringutils.to_str(other2['verhex']), - }) - - listkeys = self.mainKeeper.list_keys() - self.assertDictEqual(listkeys, {'accepted': ['other1', 'other2'], - 'rejected': [], - 'pending': []}) - - allremotekeys = self.mainKeeper.read_all_remote() - self.assertDictEqual(allremotekeys, - {'other1': - {'verify': salt.utils.stringutils.to_str(other1['verhex']), - 'minion_id': 'other1', - 'acceptance': 'accepted', - 'pub': salt.utils.stringutils.to_str(other1['pubhex']) - }, - 'other2': - {'verify': salt.utils.stringutils.to_str(other2['verhex']), - 'minion_id': 'other2', - 'acceptance': 'accepted', - 'pub': salt.utils.stringutils.to_str(other2['pubhex']), - } - }) - - self.mainKeeper.delete_key(match=other1['name']) - - allkeys = self.mainKeeper.all_keys() - self.assertDictEqual(allkeys, {'accepted': ['other2'], - 'local': [self.localFilepath], - 'pending': [], - 'rejected': []}) - - remotekeys = self.mainKeeper.read_remote(other1['name']) - self.assertDictEqual(remotekeys, {}) - - remotekeys = self.mainKeeper.read_remote(other2['name']) - self.assertDictEqual(remotekeys, {'minion_id': 'other2', - 'pub': salt.utils.stringutils.to_str(other2['pubhex']), - 'verify': salt.utils.stringutils.to_str(other2['verhex'])}) - - listkeys = self.mainKeeper.list_keys() - self.assertDictEqual(listkeys, {'accepted': ['other2'], - 'rejected': [], - 'pending': []}) - - allremotekeys = self.mainKeeper.read_all_remote() - self.assertDictEqual(allremotekeys, - {'other2': - {'verify': salt.utils.stringutils.to_str(other2['verhex']), - 'minion_id': 'other2', - 'acceptance': 'accepted', - 'pub': salt.utils.stringutils.to_str(other2['pubhex']), - } - }) - - -def runOne(test): - ''' - Unittest Runner - ''' - test = BasicTestCase(test) - suite = unittest.TestSuite([test]) - unittest.TextTestRunner(verbosity=2).run(suite) - - -def runSome(): - ''' - Unittest runner - ''' - tests = [] - names = ['testAutoAccept', - 'testManualAccept', - 'testDelete'] - - tests.extend(list(list(map(BasicTestCase, names)))) - - suite = unittest.TestSuite(tests) - unittest.TextTestRunner(verbosity=2).run(suite) - - -def runAll(): - ''' - Unittest runner - ''' - suite = unittest.TestSuite() - suite.addTest(unittest.TestLoader().loadTestsFromTestCase(BasicTestCase)) - - unittest.TextTestRunner(verbosity=2).run(suite) - - -if __name__ == '__main__' and __package__ is None: - # console.reinit(verbosity=console.Wordage.concise) - - runAll() # run all unittests - - # runSome() #only run some - - # runOne('testDelete') diff --git a/salt/daemons/test/test_saltkeep.py b/salt/daemons/test/test_saltkeep.py deleted file mode 100644 index 9cf2dc0bfcf..00000000000 --- a/salt/daemons/test/test_saltkeep.py +++ /dev/null @@ -1,2236 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Tests to try out salt key.RaetKey Potentially ephemeral -''' -from __future__ import absolute_import, print_function, unicode_literals -import sys -from salt.ext.six.moves import map -# pylint: disable=blacklisted-import -if sys.version_info < (2, 7): - import unittest2 as unittest -else: - import unittest -# pylint: enable=blacklisted-import - -import os -import shutil -import socket -import stat -import tempfile -import time - -from ioflo.aid.odicting import odict -from ioflo.aid.timing import StoreTimer -from ioflo.base import storing -from ioflo.base.consoling import getConsole -console = getConsole() - -from raet import raeting, nacling -from raet.road import estating, stacking - -from salt.daemons import salting -import salt.utils.kinds as kinds -import salt.utils.stringutils - - -def setUpModule(): - console.reinit(verbosity=console.Wordage.concise) - - -def tearDownModule(): - pass - - -class BasicTestCase(unittest.TestCase): # pylint: disable=moved-test-case-class - - def setUp(self): - self.store = storing.Store(stamp=0.0) - self.timer = StoreTimer(store=self.store, duration=1.0) - - self.tempDirpath = tempfile.mkdtemp(prefix="salt", suffix='keep', dir='/tmp') - self.maxDiff = None - - def tearDown(self): - if os.path.exists(self.tempDirpath): - shutil.rmtree(self.tempDirpath) - - def createOpts(self, - role, - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.master], - dirpath='/tmp', - openMode=False, - autoAccept=True): - ''' - Create associated pki directories for stack and return opts - - os.path.join(cache, 'raet', name, 'remote') - ''' - pkiDirpath = os.path.join(dirpath, 'pki', role, 'raet') - if not os.path.exists(pkiDirpath): - os.makedirs(pkiDirpath) - - acceptedDirpath = os.path.join(pkiDirpath, 'accepted') - if not os.path.exists(acceptedDirpath): - os.makedirs(acceptedDirpath) - - pendingDirpath = os.path.join(pkiDirpath, 'pending') - if not os.path.exists(pendingDirpath): - os.makedirs(pendingDirpath) - - rejectedDirpath = os.path.join(pkiDirpath, 'rejected') - if not os.path.exists(rejectedDirpath): - os.makedirs(rejectedDirpath) - - localFilepath = os.path.join(pkiDirpath, 'local.key') - if os.path.exists(localFilepath): - mode = os.stat(localFilepath).st_mode - print(mode) - os.chmod(localFilepath, mode | stat.S_IWUSR | stat.S_IWUSR) - - cacheDirpath = os.path.join(dirpath, 'cache', role) - sockDirpath = os.path.join(dirpath, 'sock', role) - - opts = dict( - id=role, - pki_dir=pkiDirpath, - sock_dir=sockDirpath, - cachedir=cacheDirpath, - open_mode=openMode, - auto_accept=autoAccept, - transport='raet', - __role=kind, - ) - return opts - - def createRoadData(self, role, kind=kinds.APPL_KIND_NAMES[kinds.applKinds.master], cachedirpath=''): - ''' - Creates odict and populates with data to setup road stack - { - name: stack name local estate name - dirpath: dirpath for keep files - sighex: signing key - verhex: verify key - prihex: private key - pubhex: public key - } - ''' - data = odict() - data['name'] = "{0}_{1}".format(role, kind) - data['role'] = role - data['kind'] = kinds.APPL_KINDS[kind] # convert to integer from kind name - data['basedirpath'] = os.path.join(cachedirpath, 'raet') - signer = nacling.Signer() - data['sighex'] = signer.keyhex - data['verhex'] = signer.verhex - privateer = nacling.Privateer() - data['prihex'] = privateer.keyhex - data['pubhex'] = privateer.pubhex - - return data - - def createRoadStack(self, data, keep, uid=None, main=None, ha=None, mutable=None): - ''' - Creates stack and local estate from data with - local estate.uid = uid - stack.main = main - stack.mutable = mutable - stack.auto = auto - stack.name = data['name'] - local estate.name = data['name'] - local estate.ha = ha - - returns stack - - ''' - stack = stacking.RoadStack(store=self.store, - name=data['name'], - keep=keep, - uid=uid, - ha=ha, - main=main, - mutable=mutable, - role=data['role'], - kind=data['kind'], - sigkey=data['sighex'], - prikey=data['prihex'],) - - return stack - - def join(self, initiator, correspondent, deid=None, duration=1.0): - ''' - Utility method to do join. Call from test method. - ''' - console.terse("\nJoin Transaction **************\n") - if not initiator.remotes: - remote = initiator.addRemote(estating.RemoteEstate(stack=initiator, - fuid=0, # vacuous join - sid=0, # always 0 for join - ha=correspondent.local.ha)) - deid = remote.uid - initiator.join(uid=deid) - self.service(correspondent, initiator, duration=duration) - - def allow(self, other, main, duration=1.0): - ''' - Utility method to do allow. Call from test method. - ''' - console.terse("\nAllow Transaction **************\n") - other.allow() - self.service(main, other, duration=duration) - - def message(self, main, other, mains, others, duration=2.0): - ''' - Utility to send messages both ways - ''' - for msg in mains: - main.transmit(msg) - for msg in others: - other.transmit(msg) - - self.service(main, other, duration=duration) - - def service(self, main, other, duration=1.0): - ''' - Utility method to service queues. Call from test method. - ''' - self.timer.restart(duration=duration) - while not self.timer.expired: - other.serviceAll() - main.serviceAll() - if not (main.transactions or other.transactions): - break - self.store.advanceStamp(0.05) - time.sleep(0.05) - - def testBasic(self): - ''' - Basic keep setup for stack keep persistence load and dump in normal mode - ''' - console.terse("{0}\n".format(self.testBasic.__doc__)) - - opts = self.createOpts(role='main', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.master], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=False) - mainData = self.createRoadData(cachedirpath=opts['cachedir'], - role=opts['id'], - kind=opts['__role']) - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - - self.assertEqual(mainKeep.loadLocalData(), None) - self.assertEqual(mainKeep.loadAllRemoteData(), {}) - - main = self.createRoadStack(data=mainData, - main=True, - ha=None, # default ha is ("", raeting.RAET_PORT) - keep=mainKeep) - - console.terse("{0}\nkeep dirpath = {1}\n".format( - main.name, main.keep.dirpath)) - self.assertTrue(main.keep.dirpath.endswith( - os.path.join('main', 'raet', 'main_master'))) - self.assertTrue(main.ha, ("0.0.0.0", raeting.RAET_PORT)) - self.assertIs(main.keep.auto, raeting.AutoMode.never.value) - self.assertDictEqual(main.keep.loadLocalData(), - {'name': mainData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7530], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7530], - 'role': mainData['role'], - 'sighex': salt.utils.stringutils.to_str(mainData['sighex']), - 'prihex': salt.utils.stringutils.to_str(mainData['prihex']), - }) - - data1 = self.createRoadData(role='remote1', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - cachedirpath=opts['cachedir']) - main.addRemote(estating.RemoteEstate(stack=main, - name=data1['name'], - kind=data1['kind'], - ha=('127.0.0.1', 7532), - role=data1['role'], - verkey=data1['verhex'], - pubkey=data1['pubhex'],)) - - data2 = self.createRoadData(role='remote2', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - cachedirpath=opts['cachedir']) - main.addRemote(estating.RemoteEstate(stack=main, - name=data2['name'], - kind=data2['kind'], - ha=('127.0.0.1', 7533), - role=data2['role'], - verkey=data2['verhex'], - pubkey=data2['pubhex'],)) - - main.dumpRemotes() - - # acceptance will be pended - - self.assertDictEqual(main.keep.loadAllRemoteData(), - { - 'remote1_minion': - {'name': data1['name'], - 'uid': 2, - 'fuid': 0, - 'ha': ['127.0.0.1', 7532], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data1['kind'], - 'sid': 0, - 'joined': None, - 'role': data1['role'], - 'acceptance': 0, - 'verhex': salt.utils.stringutils.to_str(data1['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data1['pubhex']), - }, - 'remote2_minion': - {'name': data2['name'], - 'uid': 3, - 'fuid': 0, - 'ha': ['127.0.0.1', 7533], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data2['kind'], - 'sid': 0, - 'joined': None, - 'role': data2['role'], - 'acceptance': 0, - 'verhex': salt.utils.stringutils.to_str(data2['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data2['pubhex']), - } - }) - - # now recreate with saved data - main.server.close() - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - main = stacking.RoadStack(name=mainData['name'], - store=self.store, - keep=mainKeep, - main=True) - - self.assertEqual(main.local.priver.keyhex, mainData['prihex']) - self.assertEqual(main.local.signer.keyhex, mainData['sighex']) - - self.assertEqual(len(main.remotes), 2) - - # other stack - opts = self.createOpts(role='other', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=False) - otherData = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir']) - otherKeep = salting.SaltKeep(opts=opts, - basedirpath=otherData['basedirpath'], - stackname=otherData['name']) - - self.assertEqual(otherKeep.loadLocalData(), None) - self.assertEqual(otherKeep.loadAllRemoteData(), {}) - - other = self.createRoadStack(data=otherData, - main=None, - ha=("", raeting.RAET_TEST_PORT), - keep=otherKeep) - - console.terse("{0} keep dirpath = {1}\n".format( - other.name, other.keep.dirpath)) - self.assertTrue(other.keep.dirpath.endswith(os.path.join('other', 'raet', 'other_minion'))) - self.assertEqual(other.ha, ("0.0.0.0", raeting.RAET_TEST_PORT)) - self.assertIs(other.keep.auto, raeting.AutoMode.never.value) - - self.assertDictEqual(other.keep.loadLocalData(), - { - 'name': otherData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7531], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7531], - 'role': otherData['role'], - 'sighex': salt.utils.stringutils.to_str(otherData['sighex']), - 'prihex': salt.utils.stringutils.to_str(otherData['prihex']), - }) - - data3 = self.createRoadData(role='remote3', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - cachedirpath=opts['cachedir']) - other.addRemote(estating.RemoteEstate(stack=other, - name=data3['name'], - kind=data3['kind'], - ha=('127.0.0.1', 7534), - role=data3['role'], - verkey=data3['verhex'], - pubkey=data3['pubhex'],)) - - data4 = self.createRoadData(role='remote4', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - cachedirpath=opts['cachedir']) - other.addRemote(estating.RemoteEstate(stack=other, - name=data4['name'], - kind=data4['kind'], - ha=('127.0.0.1', 7535), - role=data4['role'], - verkey=data4['verhex'], - pubkey=data4['pubhex'],)) - - other.dumpRemotes() - self.assertDictEqual(other.keep.loadAllRemoteData(), - { - 'remote3_minion': - { - 'name': data3['name'], - 'uid': 2, - 'fuid': 0, - 'ha': ['127.0.0.1', 7534], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data3['kind'], - 'sid': 0, - 'joined': None, - 'role': data3['role'], - 'acceptance': 0, - 'verhex': salt.utils.stringutils.to_str(data3['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data3['pubhex']), - }, - 'remote4_minion': - { - 'name': data4['name'], - 'uid': 3, - 'fuid': 0, - 'ha': ['127.0.0.1', 7535], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data4['kind'], - 'sid': 0, - 'joined': None, - 'role': data4['role'], - 'acceptance': 0, - 'verhex': salt.utils.stringutils.to_str(data4['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data4['pubhex']), - } - }) - - main.server.close() - other.server.close() - - def testBasicOpen(self): - ''' - Basic keep setup for stack keep persistence load and dump in open mode - ''' - console.terse("{0}\n".format(self.testBasicOpen.__doc__)) - - opts = self.createOpts(role='main', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.master], - dirpath=self.tempDirpath, - openMode=True, - autoAccept=True) - mainData = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir']) - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - - self.assertEqual(mainKeep.loadLocalData(), None) - self.assertEqual(mainKeep.loadAllRemoteData(), {}) - - main = self.createRoadStack(data=mainData, - main=True, - ha=None, # default ha is ("", raeting.RAET_PORT) - keep=mainKeep) - - console.terse("{0}\nkeep dirpath = {1}\n".format( - main.name, main.keep.dirpath)) - self.assertTrue(main.keep.dirpath.endswith(os.path.join('main', 'raet', 'main_master'))) - self.assertTrue(main.ha, ("0.0.0.0", raeting.RAET_PORT)) - self.assertIs(main.keep.auto, raeting.AutoMode.always.value) - self.assertDictEqual(main.keep.loadLocalData(), { - 'name': mainData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7530], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7530], - 'role': mainData['role'], - 'sighex': salt.utils.stringutils.to_str(mainData['sighex']), - 'prihex': salt.utils.stringutils.to_str(mainData['prihex']), - }) - - data1 = self.createRoadData(role='remote1', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - cachedirpath=opts['cachedir']) - main.addRemote(estating.RemoteEstate(stack=main, - name=data1['name'], - kind=data1['kind'], - ha=('127.0.0.1', 7532), - role=data1['role'], - verkey=data1['verhex'], - pubkey=data1['pubhex'],)) - - data2 = self.createRoadData(role='remote2', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - cachedirpath=opts['cachedir']) - main.addRemote(estating.RemoteEstate(stack=main, - name=data2['name'], - kind=data2['kind'], - ha=('127.0.0.1', 7533), - role=data2['role'], - verkey=data2['verhex'], - pubkey=data2['pubhex'],)) - - main.dumpRemotes() - - self.assertDictEqual(main.keep.loadAllRemoteData(), - { - 'remote1_minion': - {'name': data1['name'], - 'uid': 2, - 'fuid': 0, - 'ha': ['127.0.0.1', 7532], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data1['kind'], - 'sid': 0, - 'joined': None, - 'role': data1['role'], - 'acceptance': 1, - 'verhex': salt.utils.stringutils.to_str(data1['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data1['pubhex']), - }, - 'remote2_minion': - {'name': data2['name'], - 'uid': 3, - 'fuid': 0, - 'ha': ['127.0.0.1', 7533], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data2['kind'], - 'sid': 0, - 'joined': None, - 'role': data2['role'], - 'acceptance': 1, - 'verhex': salt.utils.stringutils.to_str(data2['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data2['pubhex']), - } - }) - - # now recreate with saved data - main.server.close() - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - main = stacking.RoadStack(name=mainData['name'], - store=self.store, - keep=mainKeep, - main=True) - - self.assertEqual(main.local.priver.keyhex, mainData['prihex']) - self.assertEqual(main.local.signer.keyhex, mainData['sighex']) - - self.assertEqual(len(main.remotes), 2) - - # other stack - opts = self.createOpts(role='other', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - dirpath=self.tempDirpath, - openMode=True, - autoAccept=True) - otherData = self.createRoadData(role='other', - kind=opts['__role'], - cachedirpath=opts['cachedir']) - otherKeep = salting.SaltKeep(opts=opts, - basedirpath=otherData['basedirpath'], - stackname=otherData['name']) - - self.assertEqual(otherKeep.loadLocalData(), None) - self.assertEqual(otherKeep.loadAllRemoteData(), {}) - - other = self.createRoadStack(data=otherData, - main=None, - ha=("", raeting.RAET_TEST_PORT), - keep=otherKeep) - - console.terse("{0} keep dirpath = {1}\n".format( - other.name, other.keep.dirpath)) - self.assertTrue(other.keep.dirpath.endswith(os.path.join('other', 'raet', 'other_minion'))) - self.assertEqual(other.ha, ("0.0.0.0", raeting.RAET_TEST_PORT)) - self.assertIs(other.keep.auto, raeting.AutoMode.always.value) - - self.assertDictEqual(other.keep.loadLocalData(), - { - 'name': otherData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7531], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7531], - 'role': otherData['role'], - 'sighex': salt.utils.stringutils.to_str(otherData['sighex']), - 'prihex': salt.utils.stringutils.to_str(otherData['prihex']), - }) - - data3 = self.createRoadData(role='remote3', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - cachedirpath=opts['cachedir']) - other.addRemote(estating.RemoteEstate(stack=other, - name=data3['name'], - kind=data3['kind'], - ha=('127.0.0.1', 7534), - role=data3['role'], - verkey=data3['verhex'], - pubkey=data3['pubhex'],)) - - data4 = self.createRoadData(role='remote4', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - cachedirpath=opts['cachedir']) - other.addRemote(estating.RemoteEstate(stack=other, - name=data4['name'], - kind=data4['kind'], - ha=('127.0.0.1', 7535), - role=data4['role'], - verkey=data4['verhex'], - pubkey=data4['pubhex'],)) - - other.dumpRemotes() - self.assertDictEqual(other.keep.loadAllRemoteData(), - { - 'remote3_minion': - { - 'name': data3['name'], - 'uid': 2, - 'fuid': 0, - 'ha': ['127.0.0.1', 7534], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data3['kind'], - 'sid': 0, - 'joined': None, - 'role': data3['role'], - 'acceptance': 1, - 'verhex': salt.utils.stringutils.to_str(data3['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data3['pubhex']), - }, - 'remote4_minion': - { - 'name': data4['name'], - 'uid': 3, - 'fuid': 0, - 'ha': ['127.0.0.1', 7535], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data4['kind'], - 'sid': 0, - 'joined': None, - 'role': data4['role'], - 'acceptance': 1, - 'verhex': salt.utils.stringutils.to_str(data4['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data4['pubhex']), - } - }) - - main.server.close() - other.server.close() - - def testBasicAuto(self): - ''' - Basic keep setup for stack keep persistence load and dump with auto accept - ''' - console.terse("{0}\n".format(self.testBasicAuto.__doc__)) - - opts = self.createOpts(role='main', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.master], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=True) - mainData = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir']) - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - - self.assertEqual(mainKeep.loadLocalData(), None) - self.assertEqual(mainKeep.loadAllRemoteData(), {}) - - main = self.createRoadStack(data=mainData, - main=True, - ha=None, # default ha is ("", raeting.RAET_PORT) - keep=mainKeep) - - console.terse("{0}\nkeep dirpath = {1}\n".format( - main.name, main.keep.dirpath)) - self.assertTrue(main.keep.dirpath.endswith(os.path.join('main', 'raet', 'main_master'))) - self.assertTrue(main.ha, ("0.0.0.0", raeting.RAET_PORT)) - self.assertIs(main.keep.auto, raeting.AutoMode.once.value) - self.assertDictEqual(main.keep.loadLocalData(), { - 'name': mainData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7530], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7530], - 'sighex': salt.utils.stringutils.to_str(mainData['sighex']), - 'prihex': salt.utils.stringutils.to_str(mainData['prihex']), - 'role': mainData['role'], - }) - - data1 = self.createRoadData(role='remote1', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - cachedirpath=opts['cachedir']) - main.addRemote(estating.RemoteEstate(stack=main, - name=data1['name'], - kind=data1['kind'], - ha=('127.0.0.1', 7532), - role=data1['role'], - verkey=data1['verhex'], - pubkey=data1['pubhex'],)) - - data2 = self.createRoadData(role='remote2', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - cachedirpath=opts['cachedir']) - main.addRemote(estating.RemoteEstate(stack=main, - name=data2['name'], - kind=data2['kind'], - ha=('127.0.0.1', 7533), - role=data2['role'], - verkey=data2['verhex'], - pubkey=data2['pubhex'],)) - - main.dumpRemotes() - - self.assertDictEqual(main.keep.loadAllRemoteData(), - { - 'remote1_minion': - { - 'name': data1['name'], - 'uid': 2, - 'fuid': 0, - 'ha': ['127.0.0.1', 7532], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data1['kind'], - 'sid': 0, - 'joined': None, - 'role': data1['role'], - 'acceptance': 1, - 'verhex': salt.utils.stringutils.to_str(data1['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data1['pubhex']), - }, - 'remote2_minion': - { - 'name': data2['name'], - 'uid': 3, - 'fuid': 0, - 'ha': ['127.0.0.1', 7533], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data2['kind'], - 'sid': 0, - 'joined': None, - 'role': data2['role'], - 'acceptance': 1, - 'verhex': salt.utils.stringutils.to_str(data2['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data2['pubhex']), - } - }) - - # now recreate with saved data - main.server.close() - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - main = stacking.RoadStack(name=mainData['name'], - store=self.store, - keep=mainKeep, - main=True) - - self.assertEqual(main.local.priver.keyhex, mainData['prihex']) - self.assertEqual(main.local.signer.keyhex, mainData['sighex']) - - self.assertEqual(len(main.remotes), 2) - - # other stack - opts = self.createOpts(role='other', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=True) - otherData = self.createRoadData(role='other', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - cachedirpath=opts['cachedir']) - otherKeep = salting.SaltKeep(opts=opts, - basedirpath=otherData['basedirpath'], - stackname=otherData['name']) - - self.assertEqual(otherKeep.loadLocalData(), None) - self.assertEqual(otherKeep.loadAllRemoteData(), {}) - - other = self.createRoadStack(data=otherData, - main=None, - ha=("", raeting.RAET_TEST_PORT), - keep=otherKeep) - - console.terse("{0} keep dirpath = {1}\n".format( - other.name, other.keep.dirpath)) - self.assertTrue(other.keep.dirpath.endswith(os.path.join('other', 'raet', 'other_minion'))) - self.assertEqual(other.ha, ("0.0.0.0", raeting.RAET_TEST_PORT)) - self.assertIs(other.keep.auto, raeting.AutoMode.once.value) - - self.assertDictEqual(other.keep.loadLocalData(), - { - 'name': otherData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7531], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7531], - 'role': otherData['role'], - 'sighex': salt.utils.stringutils.to_str(otherData['sighex']), - 'prihex': salt.utils.stringutils.to_str(otherData['prihex']), - }) - - data3 = self.createRoadData(role='remote3', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - cachedirpath=opts['cachedir']) - other.addRemote(estating.RemoteEstate(stack=other, - name=data3['name'], - kind=data3['kind'], - ha=('127.0.0.1', 7534), - role=data3['role'], - verkey=data3['verhex'], - pubkey=data3['pubhex'],)) - - data4 = self.createRoadData(role='remote4', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - cachedirpath=opts['cachedir']) - other.addRemote(estating.RemoteEstate(stack=other, - name=data4['name'], - kind=data4['kind'], - ha=('127.0.0.1', 7535), - role=data4['role'], - verkey=data4['verhex'], - pubkey=data4['pubhex'],)) - - other.dumpRemotes() - self.assertDictEqual(other.keep.loadAllRemoteData(), - { - 'remote3_minion': - { - 'name': data3['name'], - 'uid': 2, - 'fuid': 0, - 'ha': ['127.0.0.1', 7534], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data3['kind'], - 'sid': 0, - 'joined': None, - 'role': data3['role'], - 'acceptance': 1, - 'verhex': salt.utils.stringutils.to_str(data3['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data3['pubhex']), - }, - 'remote4_minion': - { - 'name': data4['name'], - 'uid': 3, - 'fuid': 0, - 'ha': ['127.0.0.1', 7535], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data4['kind'], - 'sid': 0, - 'joined': None, - 'role': data4['role'], - 'acceptance': 1, - 'verhex': salt.utils.stringutils.to_str(data4['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data4['pubhex']), - } - }) - - main.server.close() - other.server.close() - - def testBasicRole(self): - ''' - Basic keep setup for stack keep persistence load and dump with shared role - ''' - console.terse("{0}\n".format(self.testBasicRole.__doc__)) - - opts = self.createOpts(role='main', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.master], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=False) - mainData = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir'], - ) - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - - self.assertEqual(mainKeep.loadLocalData(), None) - self.assertEqual(mainKeep.loadAllRemoteData(), {}) - - main = self.createRoadStack(data=mainData, - main=True, - ha=None, # default ha is ("", raeting.RAET_PORT) - keep=mainKeep) - - console.terse("{0}\nkeep dirpath = {1}\n".format( - main.name, main.keep.dirpath)) - self.assertTrue(main.keep.dirpath.endswith(os.path.join('main', 'raet', 'main_master'))) - self.assertTrue(main.ha, ("0.0.0.0", raeting.RAET_PORT)) - self.assertIs(main.keep.auto, raeting.AutoMode.never.value) - self.assertDictEqual(main.keep.loadLocalData(), {'name': mainData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7530], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7530], - 'role': mainData['role'], - 'sighex': salt.utils.stringutils.to_str(mainData['sighex']), - 'prihex': salt.utils.stringutils.to_str(mainData['prihex']), - }) - - # add multiple remotes all with same role - data1 = self.createRoadData(role='primary', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - cachedirpath=opts['cachedir'], - ) - main.addRemote(estating.RemoteEstate(stack=main, - name=data1['name'], - kind=data1['kind'], - ha=('127.0.0.1', 7532), - role=data1['role'], - verkey=data1['verhex'], - pubkey=data1['pubhex'], - )) - - data2 = self.createRoadData(role='primary', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.caller], - cachedirpath=opts['cachedir'], - ) - main.addRemote(estating.RemoteEstate(stack=main, - name=data2['name'], - kind=data2['kind'], - ha=('127.0.0.1', 7533), - role=data2['role'], - verkey=data2['verhex'], - pubkey=data2['pubhex'], - )) - - main.dumpRemotes() - - # will save remote1 keys and reuse for remote2 - - self.assertDictEqual(main.keep.loadAllRemoteData(), - { - 'primary_minion': - { - 'name': data1['name'], - 'uid': 2, - 'fuid': 0, - 'ha': ['127.0.0.1', 7532], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data1['kind'], - 'sid': 0, - 'joined': None, - 'role': data1['role'], - 'acceptance': 0, - 'verhex': salt.utils.stringutils.to_str(data1['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data1['pubhex']), - }, - 'primary_caller': - { - 'name': data2['name'], - 'uid': 3, - 'fuid': 0, - 'ha': ['127.0.0.1', 7533], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data2['kind'], - 'sid': 0, - 'joined': None, - 'role': data1['role'], - 'acceptance': 0, - 'verhex': salt.utils.stringutils.to_str(data1['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data1['pubhex']), - } - }) - - # now recreate with saved data - main.server.close() - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - main = stacking.RoadStack(name=mainData['name'], - store=self.store, - keep=mainKeep, - main=True) - - self.assertEqual(main.local.name, mainData['name']) - self.assertEqual(main.local.uid, 1) - self.assertEqual(main.main, True) - self.assertEqual(main.local.role, mainData['role']) - self.assertIs(main.keep.auto, raeting.AutoMode.never.value) - self.assertTrue(main.ha, ("0.0.0.0", raeting.RAET_PORT)) - self.assertEqual(main.local.priver.keyhex, mainData['prihex']) - self.assertEqual(main.local.signer.keyhex, mainData['sighex']) - - self.assertEqual(len(main.remotes), 2) - for data in [data1, data2]: - remote = main.nameRemotes[data['name']] - self.assertEqual(remote.name, data['name']) - self.assertEqual(remote.role, data['role']) - self.assertEqual(remote.pubber.keyhex, data1['pubhex']) - self.assertEqual(remote.verfer.keyhex, data1['verhex']) - - main.server.close() - - def testBasicRoleOpen(self): - ''' - Basic keep setup for stack keep persistence load and dump with shared role - ''' - console.terse("{0}\n".format(self.testBasicRoleOpen.__doc__)) - - opts = self.createOpts(role='main', - dirpath=self.tempDirpath, - openMode=True, - autoAccept=True) - mainData = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir'], -) - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - - self.assertEqual(mainKeep.loadLocalData(), None) - self.assertEqual(mainKeep.loadAllRemoteData(), {}) - - main = self.createRoadStack(data=mainData, - main=True, - ha=None, # default ha is ("", raeting.RAET_PORT) - keep=mainKeep) - - console.terse("{0}\nkeep dirpath = {1}\n".format( - main.name, main.keep.dirpath)) - self.assertTrue(main.keep.dirpath.endswith(os.path.join('main', 'raet', 'main_master'))) - self.assertTrue(main.ha, ("0.0.0.0", raeting.RAET_PORT)) - self.assertIs(main.keep.auto, raeting.AutoMode.always.value) - self.assertDictEqual(main.keep.loadLocalData(), { - 'name': mainData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7530], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7530], - 'role': mainData['role'], - 'sighex': salt.utils.stringutils.to_str(mainData['sighex']), - 'prihex': salt.utils.stringutils.to_str(mainData['prihex']), - }) - - # add multiple remotes all with same role - data1 = self.createRoadData(role='primary', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - cachedirpath=opts['cachedir'],) - main.addRemote(estating.RemoteEstate(stack=main, - name=data1['name'], - kind=data1['kind'], - ha=('127.0.0.1', 7532), - role=data1['role'], - verkey=data1['verhex'], - pubkey=data1['pubhex'],)) - - data2 = self.createRoadData(role='primary', - kind='syndic', - cachedirpath=opts['cachedir'],) - main.addRemote(estating.RemoteEstate(stack=main, - name=data2['name'], - kind=data2['kind'], - ha=('127.0.0.1', 7533), - role=data2['role'], - verkey=data2['verhex'], - pubkey=data2['pubhex'],)) - - main.dumpRemotes() # second one keys will clobber first one keys - - self.assertDictEqual(main.keep.loadAllRemoteData(), - { - 'primary_minion': - { - 'name': data1['name'], - 'uid': 2, - 'fuid': 0, - 'ha': ['127.0.0.1', 7532], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data1['kind'], - 'sid': 0, - 'joined': None, - 'role': data1['role'], - 'acceptance': 1, - 'verhex': salt.utils.stringutils.to_str(data2['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data2['pubhex']), - }, - 'primary_syndic': - { - 'name': data2['name'], - 'uid': 3, - 'fuid': 0, - 'ha': ['127.0.0.1', 7533], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data2['kind'], - 'sid': 0, - 'joined': None, - 'role': data2['role'], - 'acceptance': 1, - 'verhex': salt.utils.stringutils.to_str(data2['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data2['pubhex']), - } - }) - - # now recreate with saved data - main.server.close() - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - main = stacking.RoadStack(name=mainData['name'], - store=self.store, - keep=mainKeep, - main=True) - - self.assertEqual(main.local.name, mainData['name']) - self.assertEqual(main.local.uid, 1) - self.assertEqual(main.main, True) - self.assertEqual(main.local.role, mainData['role']) - self.assertIs(main.keep.auto, raeting.AutoMode.always.value) - self.assertTrue(main.ha, ("0.0.0.0", raeting.RAET_PORT)) - self.assertEqual(main.local.priver.keyhex, mainData['prihex']) - self.assertEqual(main.local.signer.keyhex, mainData['sighex']) - - self.assertEqual(len(main.remotes), 2) - for data in [data1, data2]: - remote = main.nameRemotes[data['name']] - self.assertEqual(remote.name, data['name']) - self.assertEqual(remote.role, data['role']) - self.assertEqual(remote.pubber.keyhex, data2['pubhex']) - self.assertEqual(remote.verfer.keyhex, data2['verhex']) - - main.server.close() - - def testBasicRoleAuto(self): - ''' - Basic keep setup for stack keep persistence load and dump with shared role - ''' - console.terse("{0}\n".format(self.testBasicRoleAuto.__doc__)) - self.maxDiff = None - - opts = self.createOpts(role='main', - dirpath=self.tempDirpath, - openMode=False, - autoAccept=True) - mainData = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir'], - ) - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - - self.assertEqual(mainKeep.loadLocalData(), None) - self.assertEqual(mainKeep.loadAllRemoteData(), {}) - - main = self.createRoadStack(data=mainData, - main=True, - ha=None, # default ha is ("", raeting.RAET_PORT) - keep=mainKeep) - - console.terse("{0}\nkeep dirpath = {1}\n".format( - main.name, main.keep.dirpath)) - self.assertTrue(main.keep.dirpath.endswith(os.path.join('main', 'raet', 'main_master'))) - self.assertTrue(main.ha, ("0.0.0.0", raeting.RAET_PORT)) - self.assertIs(main.keep.auto, raeting.AutoMode.once.value) - self.assertDictEqual(main.keep.loadLocalData(), { - 'name': mainData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7530], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7530], - 'role': mainData['role'], - 'sighex': salt.utils.stringutils.to_str(mainData['sighex']), - 'prihex': salt.utils.stringutils.to_str(mainData['prihex']), - }) - - # add multiple remotes all with same role but different keys - data1 = self.createRoadData(role='primary', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - cachedirpath=opts['cachedir'], - ) - main.addRemote(estating.RemoteEstate(stack=main, - name=data1['name'], - kind=data1['kind'], - ha=('127.0.0.1', 7532), - role=data1['role'], - verkey=data1['verhex'], - pubkey=data1['pubhex'], - )) - - data2 = self.createRoadData(role='primary', - kind='syndic', - cachedirpath=opts['cachedir'], - ) - main.addRemote(estating.RemoteEstate(stack=main, - name=data2['name'], - kind=data2['kind'], - ha=('127.0.0.1', 7533), - role=data2['role'], - verkey=data2['verhex'], - pubkey=data2['pubhex'], - )) - - main.dumpRemotes() - - # remote2 keys will be rejected since keys do not match remote1 but same role - # upon reloading the keys for remote2 will be those from remote1 - - self.assertDictEqual(main.keep.loadAllRemoteData(), - { - 'primary_minion': - { - 'name': data1['name'], - 'uid': 2, - 'fuid': 0, - 'ha': ['127.0.0.1', 7532], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data1['kind'], - 'sid': 0, - 'joined': None, - 'role': data1['role'], - 'acceptance': 1, - 'verhex': salt.utils.stringutils.to_str(data1['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data1['pubhex']), - }, - 'primary_syndic': - { - 'name': data2['name'], - 'uid': 3, - 'fuid': 0, - 'ha': ['127.0.0.1', 7533], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'main': False, - 'kind': data2['kind'], - 'sid': 0, - 'joined': None, - 'role': data2['role'], - 'acceptance': 1, - 'verhex': salt.utils.stringutils.to_str(data1['verhex']), - 'pubhex': salt.utils.stringutils.to_str(data1['pubhex']), - } - }) - - # now recreate with saved data - main.server.close() - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - main = stacking.RoadStack(name=mainData['name'], - store=self.store, - keep=mainKeep, - main=True) - - self.assertEqual(main.local.name, mainData['name']) - self.assertEqual(main.local.uid, 1) - self.assertEqual(main.main, True) - self.assertEqual(main.local.role, mainData['role']) - self.assertIs(main.keep.auto, raeting.AutoMode.once.value) - self.assertTrue(main.ha, ("0.0.0.0", raeting.RAET_PORT)) - self.assertEqual(main.local.priver.keyhex, mainData['prihex']) - self.assertEqual(main.local.signer.keyhex, mainData['sighex']) - - self.assertEqual(len(main.remotes), 2) - for data in [data1, data2]: - remote = main.nameRemotes[data['name']] - self.assertEqual(remote.name, data['name']) - self.assertEqual(remote.role, data['role']) - self.assertEqual(remote.acceptance, 1) - self.assertEqual(remote.pubber.keyhex, data1['pubhex']) - self.assertEqual(remote.verfer.keyhex, data1['verhex']) - - main.server.close() - - def testBootstrapNever(self): - ''' - Bootstap to allowed with never mode on main - ''' - console.terse("{0}\n".format(self.testBootstrapNever.__doc__)) - - opts = self.createOpts(role='main', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.master], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=False) - mainData = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir']) - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - - self.assertEqual(mainKeep.loadLocalData(), None) - self.assertEqual(mainKeep.loadAllRemoteData(), {}) - - main = self.createRoadStack(data=mainData, - main=True, - ha=None, # default ha is ("", raeting.RAET_PORT) - keep=mainKeep) - - console.terse("{0}\nkeep dirpath = {1}\n".format( - main.name, main.keep.dirpath)) - self.assertTrue(main.keep.dirpath.endswith(os.path.join('main', 'raet', 'main_master'))) - self.assertTrue(main.ha, ("0.0.0.0", raeting.RAET_PORT)) - self.assertIs(main.keep.auto, raeting.AutoMode.never.value) - self.assertDictEqual(main.keep.loadLocalData(), { - 'name': mainData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7530], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7530], - 'role': mainData['role'], - 'sighex': salt.utils.stringutils.to_str(mainData['sighex']), - 'prihex': salt.utils.stringutils.to_str(mainData['prihex']), - }) - - opts = self.createOpts(role='other', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=True) - otherData = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir']) - otherKeep = salting.SaltKeep(opts=opts, - basedirpath=otherData['basedirpath'], - stackname=otherData['name']) - - self.assertEqual(otherKeep.loadLocalData(), None) - self.assertEqual(otherKeep.loadAllRemoteData(), {}) - - other = self.createRoadStack(data=otherData, - main=None, - ha=("", raeting.RAET_TEST_PORT), - keep=otherKeep) - - console.terse("{0} keep dirpath = {1}\n".format( - other.name, other.keep.dirpath)) - self.assertTrue(other.keep.dirpath.endswith(os.path.join('other', 'raet', 'other_minion'))) - self.assertEqual(other.ha, ("0.0.0.0", raeting.RAET_TEST_PORT)) - self.assertIs(other.keep.auto, raeting.AutoMode.once.value) - self.assertDictEqual(other.keep.loadLocalData(), - { - 'name': otherData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7531], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7531], - 'role': otherData['role'], - 'sighex': salt.utils.stringutils.to_str(otherData['sighex']), - 'prihex': salt.utils.stringutils.to_str(otherData['prihex']), - }) - - self.join(other, main) - self.assertEqual(len(main.transactions), 1) # pending - main.keep.acceptRemote(main.nameRemotes[other.local.name]) - self.service(main, other, duration=1.0) - - self.assertEqual(len(main.transactions), 0) - remote = next(iter(main.remotes.values())) - self.assertTrue(remote.joined) - self.assertEqual(len(other.transactions), 0) - remote = next(iter(other.remotes.values())) - self.assertTrue(remote.joined) - - self.allow(other, main) - self.assertEqual(len(main.transactions), 0) - remote = next(iter(main.remotes.values())) - self.assertTrue(remote.allowed) - self.assertEqual(len(other.transactions), 0) - remote = next(iter(other.remotes.values())) - self.assertTrue(remote.allowed) - - for remote in main.remotes.values(): - path = os.path.join(main.keep.remotedirpath, - "{0}.{1}.{2}".format(main.keep.prefix, remote.name, main.keep.ext)) - self.assertTrue(os.path.exists(path)) - - # now delete a key and see if road keep file is also deleted - main.keep.saltRaetKey.delete_key(match=other.local.role) - remote = main.remotes[2] - path = os.path.join(main.keep.remotedirpath, - "{0}.{1}.{2}".format(main.keep.prefix, remote.name, main.keep.ext)) - self.assertFalse(os.path.exists(path)) - - for stack in [main, other]: - stack.server.close() - stack.clearAllKeeps() - - def testBootstrapOpen(self): - ''' - Bootstap to allowed with open mode on main - ''' - console.terse("{0}\n".format(self.testBootstrapOpen.__doc__)) - - opts = self.createOpts(role='main', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.master], - dirpath=self.tempDirpath, - openMode=True, - autoAccept=True) - mainData = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir']) - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - - self.assertEqual(mainKeep.loadLocalData(), None) - self.assertEqual(mainKeep.loadAllRemoteData(), {}) - - main = self.createRoadStack(data=mainData, - main=True, - ha=None, # default ha is ("", raeting.RAET_PORT) - keep=mainKeep) - - console.terse("{0}\nkeep dirpath = {1}\n".format( - main.name, main.keep.dirpath)) - self.assertTrue(main.keep.dirpath.endswith(os.path.join('main', 'raet', 'main_master'))) - self.assertTrue(main.ha, ("0.0.0.0", raeting.RAET_PORT)) - self.assertIs(main.keep.auto, raeting.AutoMode.always.value) - self.assertDictEqual(main.keep.loadLocalData(), { - 'name': mainData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7530], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7530], - 'role': mainData['role'], - 'sighex': salt.utils.stringutils.to_str(mainData['sighex']), - 'prihex': salt.utils.stringutils.to_str(mainData['prihex']), - }) - - opts = self.createOpts(role='other', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=True) - otherData = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir']) - otherKeep = salting.SaltKeep(opts=opts, - basedirpath=otherData['basedirpath'], - stackname=otherData['name']) - - self.assertEqual(otherKeep.loadLocalData(), None) - self.assertEqual(otherKeep.loadAllRemoteData(), {}) - - other = self.createRoadStack(data=otherData, - main=None, - ha=("", raeting.RAET_TEST_PORT), - keep=otherKeep) - - console.terse("{0} keep dirpath = {1}\n".format( - other.name, other.keep.dirpath)) - self.assertTrue(other.keep.dirpath.endswith(os.path.join('other', 'raet', 'other_minion'))) - self.assertEqual(other.ha, ("0.0.0.0", raeting.RAET_TEST_PORT)) - self.assertIs(other.keep.auto, raeting.AutoMode.once.value) - self.assertDictEqual(other.keep.loadLocalData(), - { - 'name': otherData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7531], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7531], - 'role': otherData['role'], - 'sighex': salt.utils.stringutils.to_str(otherData['sighex']), - 'prihex': salt.utils.stringutils.to_str(otherData['prihex']), - }) - - self.join(other, main) - self.assertEqual(len(main.transactions), 0) - remote = next(iter(main.remotes.values())) - self.assertTrue(remote.joined) - self.assertEqual(len(other.transactions), 0) - remote = next(iter(other.remotes.values())) - self.assertTrue(remote.joined) - - self.allow(other, main) - self.assertEqual(len(main.transactions), 0) - remote = next(iter(main.remotes.values())) - self.assertTrue(remote.allowed) - self.assertEqual(len(other.transactions), 0) - remote = next(iter(other.remotes.values())) - self.assertTrue(remote.allowed) - - for remote in main.remotes.values(): - path = os.path.join(main.keep.remotedirpath, - "{0}.{1}.{2}".format(main.keep.prefix, remote.name, main.keep.ext)) - self.assertTrue(os.path.exists(path)) - - # now delete a key and see if road keep file is also deleted - main.keep.saltRaetKey.delete_key(match=other.local.role) - remote = main.remotes[2] - path = os.path.join(main.keep.remotedirpath, - "{0}.{1}.{2}".format(main.keep.prefix, remote.name, main.keep.ext)) - self.assertFalse(os.path.exists(path)) - - for stack in [main, other]: - stack.server.close() - stack.clearAllKeeps() - - def testBootstrapAuto(self): - ''' - Bootstap to allowed with auto accept on main - ''' - console.terse("{0}\n".format(self.testBootstrapAuto.__doc__)) - - opts = self.createOpts(role='main', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.master], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=True) - mainData = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir']) - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - - self.assertEqual(mainKeep.loadLocalData(), None) - self.assertEqual(mainKeep.loadAllRemoteData(), {}) - - main = self.createRoadStack(data=mainData, - main=True, - ha=None, # default ha is ("", raeting.RAET_PORT) - keep=mainKeep) - - console.terse("{0}\nkeep dirpath = {1}\n".format( - main.name, main.keep.dirpath)) - self.assertTrue(main.keep.dirpath.endswith(os.path.join('main', 'raet', 'main_master'))) - self.assertEqual(main.ha, ("0.0.0.0", raeting.RAET_PORT)) - self.assertIs(main.keep.auto, raeting.AutoMode.once.value) - self.assertDictEqual(main.keep.loadLocalData(), { - 'name': mainData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7530], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7530], - 'role': mainData['role'], - 'sighex': salt.utils.stringutils.to_str(mainData['sighex']), - 'prihex': salt.utils.stringutils.to_str(mainData['prihex']), - }) - - opts = self.createOpts(role='other', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=True) - otherData = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir']) - otherKeep = salting.SaltKeep(opts=opts, - basedirpath=otherData['basedirpath'], - stackname=otherData['name']) - - self.assertEqual(otherKeep.loadLocalData(), None) - self.assertEqual(otherKeep.loadAllRemoteData(), {}) - - other = self.createRoadStack(data=otherData, - main=None, - ha=("", raeting.RAET_TEST_PORT), - keep=otherKeep) - - console.terse("{0} keep dirpath = {1}\n".format( - other.name, other.keep.dirpath)) - self.assertTrue(other.keep.dirpath.endswith(os.path.join('other', 'raet', 'other_minion'))) - self.assertEqual(other.ha, ("0.0.0.0", raeting.RAET_TEST_PORT)) - self.assertIs(other.keep.auto, raeting.AutoMode.once.value) - self.assertDictEqual(other.keep.loadLocalData(), - { - 'name': otherData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7531], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7531], - 'sighex': salt.utils.stringutils.to_str(otherData['sighex']), - 'prihex': salt.utils.stringutils.to_str(otherData['prihex']), - 'role': otherData['role'], - }) - - self.join(other, main) - self.assertEqual(len(main.transactions), 0) - remote = next(iter(main.remotes.values())) - self.assertTrue(remote.joined) - self.assertEqual(len(other.transactions), 0) - remote = next(iter(other.remotes.values())) - self.assertTrue(remote.joined) - - self.allow(other, main) - self.assertEqual(len(main.transactions), 0) - remote = next(iter(main.remotes.values())) - self.assertTrue(remote.allowed) - self.assertEqual(len(other.transactions), 0) - remote = next(iter(other.remotes.values())) - self.assertTrue(remote.allowed) - - for remote in main.remotes.values(): - path = os.path.join(main.keep.remotedirpath, - "{0}.{1}.{2}".format(main.keep.prefix, remote.name, main.keep.ext)) - self.assertTrue(os.path.exists(path)) - - # now delete a key and see if road keep file is also deleted - main.keep.saltRaetKey.delete_key(match=other.local.role) - remote = main.remotes[2] - path = os.path.join(main.keep.remotedirpath, - "{0}.{1}.{2}".format(main.keep.prefix, remote.name, main.keep.ext)) - self.assertFalse(os.path.exists(path)) - - for stack in [main, other]: - stack.server.close() - stack.clearAllKeeps() - - def testBootstrapRoleNever(self): - ''' - Bootstap to allowed with multiple remotes using same role with never main - ''' - console.terse("{0}\n".format(self.testBootstrapRoleNever.__doc__)) - - opts = self.createOpts(role='main', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.master], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=False) - mainData = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir']) - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - - self.assertEqual(mainKeep.loadLocalData(), None) - self.assertEqual(mainKeep.loadAllRemoteData(), {}) - - main = self.createRoadStack(data=mainData, - main=True, - ha=None, # default ha is ("", raeting.RAET_PORT) - keep=mainKeep) - - console.terse("{0}\nkeep dirpath = {1}\n".format( - main.name, main.keep.dirpath)) - self.assertTrue(main.keep.dirpath.endswith(os.path.join('main', 'raet', 'main_master'))) - self.assertTrue(main.ha, ("0.0.0.0", raeting.RAET_PORT)) - self.assertIs(main.keep.auto, raeting.AutoMode.never.value) - self.assertDictEqual(main.keep.loadLocalData(), { - 'name': mainData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7530], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7530], - 'role': mainData['role'], - 'sighex': salt.utils.stringutils.to_str(mainData['sighex']), - 'prihex': salt.utils.stringutils.to_str(mainData['prihex']), - }) - - opts = self.createOpts(role='primary', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=True) - other1Data = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir']) - other1Keep = salting.SaltKeep(opts=opts, - basedirpath=other1Data['basedirpath'], - stackname=other1Data['name']) - - self.assertEqual(other1Keep.loadLocalData(), None) - self.assertEqual(other1Keep.loadAllRemoteData(), {}) - - other1 = self.createRoadStack(data=other1Data, - main=None, - ha=("", raeting.RAET_TEST_PORT), - keep=other1Keep) - - console.terse("{0} keep dirpath = {1}\n".format( - other1.name, other1.keep.dirpath)) - self.assertTrue(other1.keep.dirpath.endswith(os.path.join('primary', 'raet', 'primary_minion'))) - self.assertEqual(other1.ha, ("0.0.0.0", raeting.RAET_TEST_PORT)) - self.assertIs(other1.keep.auto, raeting.AutoMode.once.value) - self.assertDictEqual(other1.keep.loadLocalData(), - { - 'name': other1Data['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7531], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7531], - 'role': other1Data['role'], - 'sighex': salt.utils.stringutils.to_str(other1Data['sighex']), - 'prihex': salt.utils.stringutils.to_str(other1Data['prihex']), - }) - - self.join(other1, main) - self.assertEqual(len(main.transactions), 1) # pending - main.keep.acceptRemote(main.nameRemotes[other1.local.name]) - self.service(main, other1, duration=1.0) - - self.assertEqual(len(main.transactions), 0) - remote = next(iter(main.remotes.values())) - self.assertTrue(remote.joined) - self.assertEqual(len(other1.transactions), 0) - remote = next(iter(other1.remotes.values())) - self.assertTrue(remote.joined) - - self.allow(other1, main) - self.assertEqual(len(main.transactions), 0) - remote = next(iter(main.remotes.values())) - self.assertTrue(remote.allowed) - self.assertEqual(len(other1.transactions), 0) - remote = next(iter(other1.remotes.values())) - self.assertTrue(remote.allowed) - - for remote in main.remotes.values(): - path = os.path.join(main.keep.remotedirpath, - "{0}.{1}.{2}".format(main.keep.prefix, remote.name, main.keep.ext)) - self.assertTrue(os.path.exists(path)) - - # create other2 stack but use same role but different keys as other1 - opts = self.createOpts(role='primary', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.caller], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=True) - other2Data = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir']) - other2Keep = salting.SaltKeep(opts=opts, - basedirpath=other2Data['basedirpath'], - stackname=other2Data['name']) - - self.assertEqual(other2Keep.loadLocalData(), None) - self.assertEqual(other2Keep.loadAllRemoteData(), {}) - - other2 = self.createRoadStack(data=other2Data, - main=None, - ha=("", 7532), - keep=other2Keep) - - console.terse("{0} keep dirpath = {1}\n".format( - other2.name, other2.keep.dirpath)) - self.assertTrue(other2.keep.dirpath.endswith(os.path.join('primary', 'raet', 'primary_caller'))) - self.assertEqual(other2.ha, ("0.0.0.0", 7532)) - self.assertIs(other2.keep.auto, raeting.AutoMode.once.value) - self.assertDictEqual(other2.keep.loadLocalData(), - { - 'name': other2Data['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7532], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7532], - 'sighex': salt.utils.stringutils.to_str(other2Data['sighex']), - 'prihex': salt.utils.stringutils.to_str(other2Data['prihex']), - 'role': other2Data['role'], - }) - - # should not join since role same but keys different - self.join(other2, main) - self.assertEqual(len(main.transactions), 0) # rejected since not same keys - self.assertEqual(len(other2.remotes), 0) - self.assertEqual(len(main.remotes), 1) - #main.removeRemote(main.nameRemotes[other2.local.name], clear=True) - other2.server.close() - other2.keep.clearAllDir() - path = os.path.join(main.keep.remotedirpath, - "{0}.{1}.{2}".format(main.keep.prefix, other2.local.name, main.keep.ext)) - self.assertFalse(os.path.exists(path)) - other2.keep.clearRoleDir() - self.assertFalse(os.path.exists(opts['pki_dir'])) - #shutil.rmtree(opts['pki_dir']) - - # recreate other2 stack but use same role and same keys as other1 - opts = self.createOpts(role='primary', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.caller], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=True) - other2Data = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir']) - other2Data['sighex'] = other1Data['sighex'] - other2Data['prihex'] = other1Data['prihex'] - other2Keep = salting.SaltKeep(opts=opts, - basedirpath=other2Data['basedirpath'], - stackname=other2Data['name']) - - self.assertEqual(other2Keep.loadLocalData(), None) - self.assertEqual(other2Keep.loadAllRemoteData(), {}) - - other2 = self.createRoadStack(data=other2Data, - main=None, - ha=("", 7532), - keep=other2Keep) - - console.terse("{0} keep dirpath = {1}\n".format( - other2.name, other2.keep.dirpath)) - self.assertTrue(other2.keep.dirpath.endswith(os.path.join('primary', 'raet', 'primary_caller'))) - self.assertEqual(other2.ha, ("0.0.0.0", 7532)) - self.assertIs(other2.keep.auto, raeting.AutoMode.once.value) - self.assertDictEqual(other2.keep.loadLocalData(), - { - 'name': other2Data['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7532], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7532], - 'role': other2Data['role'], - 'sighex': salt.utils.stringutils.to_str(other1Data['sighex']), - 'prihex': salt.utils.stringutils.to_str(other1Data['prihex']), - }) - - # should join since same role and keys - self.join(other2, main) - - self.assertEqual(len(main.transactions), 0) - remote = next(iter(main.remotes.values())) - self.assertTrue(remote.joined) - self.assertEqual(len(other2.transactions), 0) - remote = next(iter(other2.remotes.values())) - self.assertTrue(remote.joined) - - self.allow(other2, main) - self.assertEqual(len(main.transactions), 0) - remote = next(iter(main.remotes.values())) - self.assertTrue(remote.allowed) - self.assertEqual(len(other2.transactions), 0) - remote = next(iter(other2.remotes.values())) - self.assertTrue(remote.allowed) - - for remote in main.remotes.values(): - path = os.path.join(main.keep.remotedirpath, - "{0}.{1}.{2}".format(main.keep.prefix, remote.name, main.keep.ext)) - self.assertTrue(os.path.exists(path)) - - # now delete a key and see if both road keep file are also deleted - main.keep.saltRaetKey.delete_key(match=other1.local.role) - remote = main.remotes[2] - path = os.path.join(main.keep.remotedirpath, - "{0}.{1}.{2}".format(main.keep.prefix, remote.name, main.keep.ext)) - self.assertFalse(os.path.exists(path)) - remote = main.remotes[4] - path = os.path.join(main.keep.remotedirpath, - "{0}.{1}.{2}".format(main.keep.prefix, remote.name, main.keep.ext)) - self.assertFalse(os.path.exists(path)) - - for stack in [main, other1, other2]: - stack.server.close() - stack.clearAllKeeps() - - def testBootstrapRoleAuto(self): - ''' - Bootstap to allowed with multiple remotes using same role with auto main - ''' - console.terse("{0}\n".format(self.testBootstrapRoleAuto.__doc__)) - - opts = self.createOpts(role='main', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.master], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=True) - mainData = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir']) - mainKeep = salting.SaltKeep(opts=opts, - basedirpath=mainData['basedirpath'], - stackname=mainData['name']) - - self.assertEqual(mainKeep.loadLocalData(), None) - self.assertEqual(mainKeep.loadAllRemoteData(), {}) - - main = self.createRoadStack(data=mainData, - main=True, - ha=None, # default ha is ("", raeting.RAET_PORT) - keep=mainKeep) - - console.terse("{0}\nkeep dirpath = {1}\n".format( - main.name, main.keep.dirpath)) - self.assertTrue(main.keep.dirpath.endswith(os.path.join('main', 'raet', 'main_master'))) - self.assertTrue(main.ha, ("0.0.0.0", raeting.RAET_PORT)) - self.assertIs(main.keep.auto, raeting.AutoMode.once.value) - self.assertDictEqual(main.keep.loadLocalData(), { - 'name': mainData['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7530], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7530], - 'role': mainData['role'], - 'sighex': salt.utils.stringutils.to_str(mainData['sighex']), - 'prihex': salt.utils.stringutils.to_str(mainData['prihex']), - }) - - opts = self.createOpts(role='primary', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=True) - other1Data = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir']) - other1Keep = salting.SaltKeep(opts=opts, - basedirpath=other1Data['basedirpath'], - stackname=other1Data['name']) - - self.assertEqual(other1Keep.loadLocalData(), None) - self.assertEqual(other1Keep.loadAllRemoteData(), {}) - - other1 = self.createRoadStack(data=other1Data, - main=None, - ha=("", raeting.RAET_TEST_PORT), - keep=other1Keep) - - console.terse("{0} keep dirpath = {1}\n".format( - other1.name, other1.keep.dirpath)) - self.assertTrue(other1.keep.dirpath.endswith(os.path.join('primary', 'raet', 'primary_minion'))) - self.assertEqual(other1.ha, ("0.0.0.0", raeting.RAET_TEST_PORT)) - self.assertIs(other1.keep.auto, raeting.AutoMode.once.value) - self.assertDictEqual(other1.keep.loadLocalData(), - { - 'name': other1Data['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7531], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7531], - 'role': other1Data['role'], - 'sighex': salt.utils.stringutils.to_str(other1Data['sighex']), - 'prihex': salt.utils.stringutils.to_str(other1Data['prihex']), - }) - - self.join(other1, main) - self.assertEqual(len(main.transactions), 0) - remote = next(iter(main.remotes.values())) - self.assertTrue(remote.joined) - self.assertEqual(len(other1.transactions), 0) - remote = next(iter(other1.remotes.values())) - self.assertTrue(remote.joined) - - self.allow(other1, main) - self.assertEqual(len(main.transactions), 0) - remote = next(iter(main.remotes.values())) - self.assertTrue(remote.allowed) - self.assertEqual(len(other1.transactions), 0) - remote = next(iter(other1.remotes.values())) - self.assertTrue(remote.allowed) - - for remote in main.remotes.values(): - path = os.path.join(main.keep.remotedirpath, - "{0}.{1}.{2}".format(main.keep.prefix, remote.name, main.keep.ext)) - self.assertTrue(os.path.exists(path)) - - # create other2 stack but use same role and different keys as other1 - opts = self.createOpts(role='primary', - kind=kinds.APPL_KIND_NAMES[kinds.applKinds.caller], - dirpath=self.tempDirpath, - openMode=False, - autoAccept=True) - other2Data = self.createRoadData(role=opts['id'], - kind=opts['__role'], - cachedirpath=opts['cachedir']) - other2Data['sighex'] = other1Data['sighex'] - other2Data['prihex'] = other1Data['prihex'] - - other2Keep = salting.SaltKeep(opts=opts, - basedirpath=other2Data['basedirpath'], - stackname=other2Data['name']) - - self.assertEqual(other2Keep.loadLocalData(), None) - self.assertEqual(other2Keep.loadAllRemoteData(), {}) - - other2 = self.createRoadStack(data=other2Data, - main=None, - ha=("", 7532), - keep=other2Keep) - - console.terse("{0} keep dirpath = {1}\n".format( - other2.name, other2.keep.dirpath)) - self.assertTrue(other2.keep.dirpath.endswith(os.path.join('primary', 'raet', 'primary_caller'))) - self.assertEqual(other2.ha, ("0.0.0.0", 7532)) - self.assertIs(other2.keep.auto, raeting.AutoMode.once.value) - self.assertDictEqual(other2.keep.loadLocalData(), - { - 'name': other2Data['name'], - 'uid': 1, - 'ha': ['127.0.0.1', 7532], - 'iha': None, - 'natted': None, - 'fqdn': socket.getfqdn('127.0.0.1'), - 'dyned': None, - 'sid': 0, - 'puid': 1, - 'aha': ['0.0.0.0', 7532], - 'role': other2Data['role'], - 'sighex': salt.utils.stringutils.to_str(other2Data['sighex']), - 'prihex': salt.utils.stringutils.to_str(other2Data['prihex']), - }) - - # should join since open mode - self.join(other2, main) - - self.assertEqual(len(main.transactions), 0) - remote = next(iter(main.remotes.values())) - self.assertTrue(remote.joined) - self.assertEqual(len(other2.transactions), 0) - remote = next(iter(other2.remotes.values())) - self.assertTrue(remote.joined) - - self.allow(other2, main) - self.assertEqual(len(main.transactions), 0) - remote = next(iter(main.remotes.values())) - self.assertTrue(remote.allowed) - self.assertEqual(len(other2.transactions), 0) - remote = next(iter(other2.remotes.values())) - self.assertTrue(remote.allowed) - - for remote in main.remotes.values(): - path = os.path.join(main.keep.remotedirpath, - "{0}.{1}.{2}".format(main.keep.prefix, remote.name, main.keep.ext)) - self.assertTrue(os.path.exists(path)) - - # now delete a key and see if both road keep file are also deleted - main.keep.saltRaetKey.delete_key(match=other1.local.role) - remote = main.remotes[2] - path = os.path.join(main.keep.remotedirpath, - "{0}.{1}.{2}".format(main.keep.prefix, remote.name, main.keep.ext)) - self.assertFalse(os.path.exists(path)) - remote = main.remotes[3] - path = os.path.join(main.keep.remotedirpath, - "{0}.{1}.{2}".format(main.keep.prefix, remote.name, main.keep.ext)) - self.assertFalse(os.path.exists(path)) - - for stack in [main, other1, other2]: - stack.server.close() - stack.clearAllKeeps() - - -def runOne(test): - ''' - Unittest Runner - ''' - test = BasicTestCase(test) - suite = unittest.TestSuite([test]) - unittest.TextTestRunner(verbosity=2).run(suite) - - -def runSome(): - ''' - Unittest runner - ''' - tests = [] - names = ['testBasic', - 'testBasicOpen', - 'testBasicAuto', - 'testBasicRole', - 'testBasicRoleOpen', - 'testBasicRoleAuto', - 'testBootstrapNever', - 'testBootstrapOpen', - 'testBootstrapAuto', - 'testBootstrapRoleNever', - 'testBootstrapRoleAuto', - ] - - tests.extend(list(map(BasicTestCase, names))) - - suite = unittest.TestSuite(tests) - unittest.TextTestRunner(verbosity=2).run(suite) - - -def runAll(): - ''' - Unittest runner - ''' - suite = unittest.TestSuite() - suite.addTest(unittest.TestLoader().loadTestsFromTestCase(BasicTestCase)) - - unittest.TextTestRunner(verbosity=2).run(suite) - - -if __name__ == '__main__' and __package__ is None: - - #console.reinit(verbosity=console.Wordage.concise) - - runAll() # run all unittests - - #runSome() # only run some - - #runOne('testBootstrapRoleAuto') diff --git a/salt/daemons/test/test_stats.py b/salt/daemons/test/test_stats.py deleted file mode 100644 index 6e23d0fdfb3..00000000000 --- a/salt/daemons/test/test_stats.py +++ /dev/null @@ -1,734 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Raet Ioflo Behavior Unittests -''' -from __future__ import absolute_import, print_function, unicode_literals -import sys -from salt.ext.six.moves import map -import importlib -# pylint: disable=blacklisted-import -if sys.version_info < (2, 7): - import unittest2 as unittest -else: - import unittest -# pylint: enable=blacklisted-import -import time - -from ioflo.base.consoling import getConsole -console = getConsole() -from ioflo.aid.odicting import odict -from ioflo.test import testing - -from raet.abiding import ns2u -from raet.lane.stacking import LaneStack -from raet.road.stacking import RoadStack - -import salt.utils.stringutils -from salt.utils.event import tagify - - -def setUpModule(): - console.reinit(verbosity=console.Wordage.concise) - - -def tearDownModule(): - pass - - -class StatsEventerTestCase(testing.FrameIofloTestCase): - ''' - Test case for Salt Raet Stats Eventer Master and Minion deeds - ''' - - def setUp(self): - ''' - Call super if override so House Framer and Frame are setup correctly - ''' - behaviors = ['salt.daemons.flo', 'salt.daemons.test.plan'] - for behavior in behaviors: - mod = importlib.import_module(behavior) - super(StatsEventerTestCase, self).setUp() - - def tearDown(self): - ''' - Call super if override so House Framer and Frame are torn down correctly - ''' - super(StatsEventerTestCase, self).tearDown() - - def testMasterContextSetup(self): - ''' - Test the context setup procedure used in all the consequence tests works as expected - This test intended to avoid some checks in other tests - ''' - console.terse("{0}\n".format(self.testMasterContextSetup.__doc__)) - - act = self.addEnterDeed("TestOptsSetupMaster") - self.assertIn(act, self.frame.enacts) - self.assertEqual(act.actor, "TestOptsSetupMaster") - act = self.addEnterDeed("SaltRaetManorLaneSetup") - self.assertIn(act, self.frame.enacts) - self.assertEqual(act.actor, "SaltRaetManorLaneSetup") - act = self.addEnterDeed("SaltRaetRoadStackSetup") - self.assertIn(act, self.frame.enacts) - self.assertEqual(act.actor, "SaltRaetRoadStackSetup") - act = self.addEnterDeed("StatsMasterTestSetup") - self.assertIn(act, self.frame.enacts) - self.assertEqual(act.actor, "StatsMasterTestSetup") - - act = self.addRecurDeed("SaltRaetStatsEventer") - self.assertIn(act, self.frame.reacts) - self.assertEqual(act.actor, "SaltRaetStatsEventer") - - self.resolve() # resolve House, Framer, Frame, Acts, Actors - - self.frame.enter() - self.assertDictEqual( - act.actor.Ioinits, - {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'stats_req': salt.utils.stringutils.to_str('.salt.stats.event_req'), - 'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'), - 'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack')}) - - self.assertTrue(hasattr(act.actor, 'opts')) - self.assertTrue(hasattr(act.actor, 'stats_req')) - self.assertTrue(hasattr(act.actor, 'lane_stack')) - self.assertTrue(hasattr(act.actor, 'road_stack')) - self.assertIsInstance(act.actor.lane_stack.value, LaneStack) - self.assertIsInstance(act.actor.road_stack.value, RoadStack) - self.frame.recur() - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - act.actor.road_stack.value.server.close() - - def testMasterRoadStats(self): - ''' - Test Master Road Stats request (A1) - ''' - console.terse("{0}\n".format(self.testMasterRoadStats.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("SaltRaetRoadStackSetup") - self.addEnterDeed("StatsMasterTestSetup") - act = self.addRecurDeed("SaltRaetStatsEventerMaster") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add a test stat key-value - roadStack = self.store.fetch('.salt.road.manor.stack') - laneStack = self.store.fetch('.salt.lane.manor.stack') - roadStack.value.stats['test_stats_event'] = 111 - # ensure stats are equal to expected - self.assertDictEqual(roadStack.value.stats, {'test_stats_event': 111}) - self.assertDictEqual(laneStack.value.stats, {}) - - # add stats request - testStack = self.store.fetch('.salt.test.lane.stack').value - statsReq = self.store.fetch('.salt.stats.event_req').value - tag = tagify('road', 'stats') - # road stats request - statsReq.append({'route': {'dst': (None, None, 'stats_req'), - 'src': (None, testStack.local.name, None)}, - 'tag': tag}) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 1) - - msg, sender = testStack.rxMsgs.popleft() - self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'test_stats_event': 111}}) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - act.actor.road_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - time.sleep(0.1) - - def testMasterLaneStats(self): - ''' - Test Master Road Stats request (A2) - ''' - console.terse("{0}\n".format(self.testMasterLaneStats.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("SaltRaetRoadStackSetup") - self.addEnterDeed("StatsMasterTestSetup") - act = self.addRecurDeed("SaltRaetStatsEventerMaster") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add a test stat key-value - roadStack = self.store.fetch('.salt.road.manor.stack') - laneStack = self.store.fetch('.salt.lane.manor.stack') - laneStack.value.stats['test_stats_event'] = 111 - # ensure stats are equal to expected - self.assertDictEqual(roadStack.value.stats, {}) - self.assertDictEqual(laneStack.value.stats, {'test_stats_event': 111}) - - # add stats request - testStack = self.store.fetch('.salt.test.lane.stack').value - statsReq = self.store.fetch('.salt.stats.event_req').value - tag = tagify('lane', 'stats') - # lane stats request - statsReq.append({'route': {'dst': (None, None, 'stats_req'), - 'src': (None, testStack.local.name, None)}, - 'tag': tag}) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 1) - - msg, sender = testStack.rxMsgs.popleft() - self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None], - 'dst': [None, None, 'event_fire']}, - 'tag': tag, - 'data': {'test_stats_event': 111}}) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - act.actor.road_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - def testMasterStatsWrongMissingTag(self): - ''' - Test Master Stats requests with unknown and missing tag (A3, A4) - ''' - console.terse("{0}\n".format(self.testMasterStatsWrongMissingTag.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("SaltRaetRoadStackSetup") - self.addEnterDeed("StatsMasterTestSetup") - act = self.addRecurDeed("SaltRaetStatsEventerMaster") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add a test stat key-value - roadStack = self.store.fetch('.salt.road.manor.stack') - laneStack = self.store.fetch('.salt.lane.manor.stack') - roadStack.value.stats['test_road_stats_event'] = 111 - laneStack.value.stats['test_lane_stats_event'] = 222 - # ensure stats are equal to expected - self.assertDictEqual(roadStack.value.stats, {'test_road_stats_event': 111}) - self.assertDictEqual(laneStack.value.stats, {'test_lane_stats_event': 222}) - - # add stats request - testStack = self.store.fetch('.salt.test.lane.stack').value - statsReq = self.store.fetch('.salt.stats.event_req').value - tag = 'salt/unknown/tag' - self.assertNotEqual(tag, tagify('lane', 'stats')) - self.assertNotEqual(tag, tagify('road', 'stats')) - # unknown tag in stats request - statsReq.append({'route': {'dst': (None, None, 'stats_req'), - 'src': (None, testStack.local.name, None)}, - 'tag': tag}) - # no tag in stats request - statsReq.append({'route': {'dst': (None, None, 'stats_req'), - 'src': (None, testStack.local.name, None)}}) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 0) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - act.actor.road_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - def testMasterStatsUnknownRemote(self): - ''' - Test Master Stats request with unknown remote (B1) - ''' - console.terse("{0}\n".format(self.testMasterStatsUnknownRemote.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("SaltRaetRoadStackSetup") - self.addEnterDeed("StatsMasterTestSetup") - act = self.addRecurDeed("SaltRaetStatsEventerMaster") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add a test stat key-value - roadStack = self.store.fetch('.salt.road.manor.stack') - laneStack = self.store.fetch('.salt.lane.manor.stack') - roadStack.value.stats['test_road_stats_event'] = 111 - laneStack.value.stats['test_lane_stats_event'] = 222 - # ensure stats are equal to expected - self.assertDictEqual(roadStack.value.stats, {'test_road_stats_event': 111}) - self.assertDictEqual(laneStack.value.stats, {'test_lane_stats_event': 222}) - - # add stats request - testStack = self.store.fetch('.salt.test.lane.stack').value - statsReq = self.store.fetch('.salt.stats.event_req').value - tag = tagify('road', 'stats') - # unknown tag in stats request - unknownName = 'unknownName' - statsReq.append({'route': {'dst': (None, None, 'stats_req'), - 'src': (None, unknownName, None)}, - 'tag': tag}) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 0) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - act.actor.road_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - def testMasterStatsNoRequest(self): - ''' - Test Master Stats no requests (nothing to do) (B2) - ''' - console.terse("{0}\n".format(self.testMasterStatsNoRequest.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMaster") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("SaltRaetRoadStackSetup") - self.addEnterDeed("StatsMasterTestSetup") - act = self.addRecurDeed("SaltRaetStatsEventerMaster") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add a test stat key-value - roadStack = self.store.fetch('.salt.road.manor.stack') - laneStack = self.store.fetch('.salt.lane.manor.stack') - roadStack.value.stats['test_road_stats_event'] = 111 - laneStack.value.stats['test_lane_stats_event'] = 222 - # ensure stats are equal to expected - self.assertDictEqual(roadStack.value.stats, {'test_road_stats_event': 111}) - self.assertDictEqual(laneStack.value.stats, {'test_lane_stats_event': 222}) - - # add stats request - testStack = self.store.fetch('.salt.test.lane.stack').value - statsReq = self.store.fetch('.salt.stats.event_req').value - # no requests - self.assertEqual(len(statsReq), 0) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 0) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - act.actor.road_stack.value.server.close() - testStack = self.store.fetch('.salt.test.lane.stack') - if testStack: - testStack.value.server.close() - - def testMinionContextSetup(self): - ''' - Test the context setup procedure used in all the consequence tests works as expected - This test intended to avoid some checks in other tests - ''' - console.terse("{0}\n".format(self.testMinionContextSetup.__doc__)) - - act = self.addEnterDeed("TestOptsSetupMinion") - self.assertIn(act, self.frame.enacts) - self.assertEqual(act.actor, "TestOptsSetupMinion") - act = self.addEnterDeed("SaltRaetManorLaneSetup") - self.assertIn(act, self.frame.enacts) - self.assertEqual(act.actor, "SaltRaetManorLaneSetup") - act = self.addEnterDeed("SaltRaetRoadStackSetup") - self.assertIn(act, self.frame.enacts) - self.assertEqual(act.actor, "SaltRaetRoadStackSetup") - act = self.addEnterDeed("StatsMinionTestSetup") - self.assertIn(act, self.frame.enacts) - self.assertEqual(act.actor, "StatsMinionTestSetup") - - act = self.addRecurDeed("SaltRaetStatsEventer") - self.assertIn(act, self.frame.reacts) - self.assertEqual(act.actor, "SaltRaetStatsEventer") - - self.resolve() # resolve House, Framer, Frame, Acts, Actors - - self.frame.enter() - self.assertDictEqual( - act.actor.Ioinits, - {'opts': salt.utils.stringutils.to_str('.salt.opts'), - 'stats_req': salt.utils.stringutils.to_str('.salt.stats.event_req'), - 'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'), - 'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack')}) - - self.assertTrue(hasattr(act.actor, 'opts')) - self.assertTrue(hasattr(act.actor, 'stats_req')) - self.assertTrue(hasattr(act.actor, 'lane_stack')) - self.assertTrue(hasattr(act.actor, 'road_stack')) - self.assertIsInstance(act.actor.lane_stack.value, LaneStack) - self.assertIsInstance(act.actor.road_stack.value, RoadStack) - self.frame.recur() - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - testStack = self.store.fetch('.salt.test.road.stack') - if testStack: - testStack.value.server.close() - act.actor.road_stack.value.server.close() - - def testMinionRoadStats(self): - ''' - Test Minion Road Stats request (A1) - ''' - console.terse("{0}\n".format(self.testMinionRoadStats.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMinion") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("SaltRaetRoadStackSetup") - self.addEnterDeed("StatsMinionTestSetup") - act = self.addRecurDeed("SaltRaetStatsEventerMinion") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add a test stat key-value - roadStack = self.store.fetch('.salt.road.manor.stack') - laneStack = self.store.fetch('.salt.lane.manor.stack') - roadStack.value.stats = odict({'test_stats_event': 111}) - laneStack.value.stats = odict() - # ensure stats are equal to expected - self.assertDictEqual(roadStack.value.stats, {'test_stats_event': 111}) - self.assertDictEqual(laneStack.value.stats, {}) - - # add stats request - testStack = self.store.fetch('.salt.test.road.stack').value - statsReq = self.store.fetch('.salt.stats.event_req').value - tag = tagify('road', 'stats') - minionName = roadStack.value.local.name - masterName = testStack.local.name - # road stats request - statsReq.append({'route': {'dst': (minionName, None, 'stats_req'), - 'src': (masterName, None, None)}, - 'tag': tag}) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 1) - - msg, sender = testStack.rxMsgs.popleft() - self.assertDictEqual(msg, {'route': {'src': [ns2u(minionName), 'manor', None], - 'dst': [ns2u(masterName), None, 'event_fire']}, - 'tag': ns2u(tag), - 'data': {'test_stats_event': 111}}) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - act.actor.road_stack.value.server.close() - testStack = self.store.fetch('.salt.test.road.stack') - if testStack: - testStack.value.server.close() - - def testMinionLaneStats(self): - ''' - Test Minion Road Stats request (A2) - ''' - console.terse("{0}\n".format(self.testMinionLaneStats.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMinion") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("SaltRaetRoadStackSetup") - self.addEnterDeed("StatsMinionTestSetup") - act = self.addRecurDeed("SaltRaetStatsEventerMinion") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add a test stat key-value - roadStack = self.store.fetch('.salt.road.manor.stack') - laneStack = self.store.fetch('.salt.lane.manor.stack') - roadStack.value.stats = odict() - laneStack.value.stats = odict({'test_stats_event': 111}) - # ensure stats are equal to expected - self.assertDictEqual(roadStack.value.stats, {}) - self.assertDictEqual(laneStack.value.stats, {'test_stats_event': 111}) - - # add stats request - testStack = self.store.fetch('.salt.test.road.stack').value - statsReq = self.store.fetch('.salt.stats.event_req').value - tag = tagify('lane', 'stats') - minionName = roadStack.value.local.name - masterName = testStack.local.name - # lane stats request - statsReq.append({'route': {'dst': (minionName, None, 'stats_req'), - 'src': (masterName, None, None)}, - 'tag': tag}) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 1) - - msg, sender = testStack.rxMsgs.popleft() - self.assertDictEqual(msg, {'route': {'src': [ns2u(minionName), 'manor', None], - 'dst': [ns2u(masterName), None, 'event_fire']}, - 'tag': ns2u(tag), - 'data': {'test_stats_event': 111}}) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - act.actor.road_stack.value.server.close() - testStack = self.store.fetch('.salt.test.road.stack') - if testStack: - testStack.value.server.close() - - def testMinionStatsWrongMissingTag(self): - ''' - Test Minion Stats requests with unknown and missing tag (A3, A4) - ''' - console.terse("{0}\n".format(self.testMinionStatsWrongMissingTag.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMinion") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("SaltRaetRoadStackSetup") - self.addEnterDeed("StatsMinionTestSetup") - act = self.addRecurDeed("SaltRaetStatsEventerMinion") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add a test stat key-value - roadStack = self.store.fetch('.salt.road.manor.stack') - laneStack = self.store.fetch('.salt.lane.manor.stack') - roadStack.value.stats = odict({'test_road_stats_event': 111}) - laneStack.value.stats = odict({'test_lane_stats_event': 222}) - # ensure stats are equal to expected - self.assertDictEqual(roadStack.value.stats, {'test_road_stats_event': 111}) - self.assertDictEqual(laneStack.value.stats, {'test_lane_stats_event': 222}) - - # add stats request - testStack = self.store.fetch('.salt.test.road.stack').value - statsReq = self.store.fetch('.salt.stats.event_req').value - tag = 'salt/unknown/tag' - self.assertNotEqual(tag, tagify('lane', 'stats')) - self.assertNotEqual(tag, tagify('road', 'stats')) - minionName = roadStack.value.local.name - masterName = testStack.local.name - # unknown tag in stats request - statsReq.append({'route': {'dst': (minionName, None, 'stats_req'), - 'src': (masterName, None, None)}, - 'tag': tag}) - # no tag in stats request - statsReq.append({'route': {'dst': (minionName, None, 'stats_req'), - 'src': (masterName, None, None)}}) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 0) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - act.actor.road_stack.value.server.close() - testStack = self.store.fetch('.salt.test.road.stack') - if testStack: - testStack.value.server.close() - - def testMinionStatsUnknownRemote(self): - ''' - Test Minion Stats request with unknown remote (B1) - ''' - console.terse("{0}\n".format(self.testMinionStatsUnknownRemote.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMinion") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("SaltRaetRoadStackSetup") - self.addEnterDeed("StatsMinionTestSetup") - act = self.addRecurDeed("SaltRaetStatsEventerMinion") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add a test stat key-value - roadStack = self.store.fetch('.salt.road.manor.stack') - laneStack = self.store.fetch('.salt.lane.manor.stack') - roadStack.value.stats = odict({'test_road_stats_event': 111}) - laneStack.value.stats = odict({'test_lane_stats_event': 222}) - # ensure stats are equal to expected - self.assertDictEqual(roadStack.value.stats, {'test_road_stats_event': 111}) - self.assertDictEqual(laneStack.value.stats, {'test_lane_stats_event': 222}) - - # add stats request - testStack = self.store.fetch('.salt.test.road.stack').value - statsReq = self.store.fetch('.salt.stats.event_req').value - tag = tagify('road', 'stats') - minionName = roadStack.value.local.name - # unknown remote (src) name in stats request - unknownName = 'unknownName' - statsReq.append({'route': {'dst': (minionName, None, 'stats_req'), - 'src': (unknownName, None, None)}, - 'tag': tag}) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 0) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - act.actor.road_stack.value.server.close() - testStack = self.store.fetch('.salt.test.road.stack') - if testStack: - testStack.value.server.close() - - def testMinionStatsNoRequest(self): - ''' - Test Minion Stats no requests (nothing to do) (B2) - ''' - console.terse("{0}\n".format(self.testMinionStatsNoRequest.__doc__)) - - # Bootstrap - self.addEnterDeed("TestOptsSetupMinion") - self.addEnterDeed("SaltRaetManorLaneSetup") - self.addEnterDeed("SaltRaetRoadStackSetup") - self.addEnterDeed("StatsMinionTestSetup") - act = self.addRecurDeed("SaltRaetStatsEventerMinion") - self.resolve() # resolve House, Framer, Frame, Acts, Actors - self.frame.enter() - - # Prepare - # add a test stat key-value - roadStack = self.store.fetch('.salt.road.manor.stack') - laneStack = self.store.fetch('.salt.lane.manor.stack') - roadStack.value.stats = odict({'test_road_stats_event': 111}) - laneStack.value.stats = odict({'test_lane_stats_event': 222}) - # ensure stats are equal to expected - self.assertDictEqual(roadStack.value.stats, {'test_road_stats_event': 111}) - self.assertDictEqual(laneStack.value.stats, {'test_lane_stats_event': 222}) - # clear lane stack remotes - - # add stats request - testStack = self.store.fetch('.salt.test.road.stack').value - statsReq = self.store.fetch('.salt.stats.event_req').value - # no request - self.assertEqual(len(statsReq), 0) - - # Test - self.frame.recur() # run in frame - - # Check - self.assertEqual(len(testStack.rxMsgs), 0) - testStack.serviceAll() - self.assertEqual(len(testStack.rxMsgs), 0) - - # Close active stacks servers - act.actor.lane_stack.value.server.close() - act.actor.road_stack.value.server.close() - testStack = self.store.fetch('.salt.test.road.stack') - if testStack: - testStack.value.server.close() - - -def runOne(test): - ''' - Unittest Runner - ''' - test = StatsEventerTestCase(test) - suite = unittest.TestSuite([test]) - unittest.TextTestRunner(verbosity=2).run(suite) - - -def runSome(): - ''' - Unittest runner - ''' - tests = [] - names = [ - 'testMasterContextSetup', - 'testMasterRoadStats', - 'testMasterLaneStats', - 'testMasterStatsWrongMissingTag', - 'testMasterStatsUnknownRemote', - 'testMasterStatsNoRequest', - 'testMinionContextSetup', - 'testMinionRoadStats', - 'testMinionLaneStats', - 'testMinionStatsWrongMissingTag', - 'testMinionStatsUnknownRemote', - 'testMinionStatsNoRequest', - ] - tests.extend(list(map(StatsEventerTestCase, names))) - suite = unittest.TestSuite(tests) - unittest.TextTestRunner(verbosity=2).run(suite) - - -def runAll(): - ''' - Unittest runner - ''' - suite = unittest.TestSuite() - suite.addTest(unittest.TestLoader().loadTestsFromTestCase(StatsEventerTestCase)) - unittest.TextTestRunner(verbosity=2).run(suite) - - -if __name__ == '__main__' and __package__ is None: - - # console.reinit(verbosity=console.Wordage.concise) - - runAll() # run all unittests - - #runSome() # only run some - - #runOne('testMasterLaneStats') diff --git a/salt/key.py b/salt/key.py index 451c3bd9d30..6cc4c33a755 100644 --- a/salt/key.py +++ b/salt/key.py @@ -7,11 +7,8 @@ used to manage salt keys directly without interfacing with the CLI. # Import python libs from __future__ import absolute_import, print_function, unicode_literals import os -import copy -import stat import shutil import fnmatch -import hashlib import logging # Import salt libs @@ -38,20 +35,11 @@ from salt.ext import six from salt.ext.six.moves import input, zip_longest # pylint: enable=import-error,no-name-in-module,redefined-builtin -# Import third party libs -try: - import msgpack -except ImportError: - pass - log = logging.getLogger(__name__) def get_key(opts): - if opts['transport'] in ('zeromq', 'tcp'): - return Key(opts) - else: - return RaetKey(opts) + return Key(opts) class KeyCLI(object): @@ -67,10 +55,7 @@ class KeyCLI(object): def __init__(self, opts): self.opts = opts self.client = salt.wheel.WheelClient(opts) - if self.opts['transport'] in ('zeromq', 'tcp'): - self.key = Key - else: - self.key = RaetKey + self.key = Key # instantiate the key object for masterless mode if not opts.get('eauth'): self.key = self.key(opts) @@ -299,71 +284,6 @@ class KeyCLI(object): return ret -class MultiKeyCLI(KeyCLI): - ''' - Manage multiple key backends from the CLI - ''' - def __init__(self, opts): - opts['__multi_key'] = True - super(MultiKeyCLI, self).__init__(opts) - # Remove the key attribute set in KeyCLI.__init__ - delattr(self, 'key') - zopts = copy.copy(opts) - ropts = copy.copy(opts) - self.keys = {} - zopts['transport'] = 'zeromq' - self.keys['ZMQ Keys'] = KeyCLI(zopts) - ropts['transport'] = 'raet' - self.keys['RAET Keys'] = KeyCLI(ropts) - - def _call_all(self, fun, *args): - ''' - Call the given function on all backend keys - ''' - for kback in self.keys: - print(kback) - getattr(self.keys[kback], fun)(*args) - - def list_status(self, status): - self._call_all('list_status', status) - - def list_all(self): - self._call_all('list_all') - - def accept(self, match, include_rejected=False, include_denied=False): - self._call_all('accept', match, include_rejected, include_denied) - - def accept_all(self, include_rejected=False, include_denied=False): - self._call_all('accept_all', include_rejected, include_denied) - - def delete(self, match): - self._call_all('delete', match) - - def delete_all(self): - self._call_all('delete_all') - - def reject(self, match, include_accepted=False, include_denied=False): - self._call_all('reject', match, include_accepted, include_denied) - - def reject_all(self, include_accepted=False, include_denied=False): - self._call_all('reject_all', include_accepted, include_denied) - - def print_key(self, match): - self._call_all('print_key', match) - - def print_all(self): - self._call_all('print_all') - - def finger(self, match, hash_type): - self._call_all('finger', match, hash_type) - - def finger_all(self, hash_type): - self._call_all('finger_all', hash_type) - - def prep_signature(self): - self._call_all('prep_signature') - - class Key(object): ''' The object that encapsulates saltkey actions @@ -599,12 +519,6 @@ class Key(object): ''' Return a dict of managed keys and what the key status are ''' - - key_dirs = [] - - # We have to differentiate between RaetKey._check_minions_directories - # and Zeromq-Keys. Raet-Keys only have three states while ZeroMQ-keys - # havd an additional 'denied' state. key_dirs = self._check_minions_directories() ret = {} @@ -962,513 +876,3 @@ class Key(object): path = os.path.join(self.opts['pki_dir'], status, key) ret[status][key] = salt.utils.crypt.pem_finger(path, sum_type=hash_type) return ret - - -class RaetKey(Key): - ''' - Manage keys from the raet backend - ''' - ACC = 'accepted' - PEND = 'pending' - REJ = 'rejected' - DEN = None - - def __init__(self, opts): - Key.__init__(self, opts) - self.auto_key = salt.daemons.masterapi.AutoKey(self.opts) - self.serial = salt.payload.Serial(self.opts) - - def _check_minions_directories(self): - ''' - Return the minion keys directory paths - ''' - accepted = os.path.join(self.opts['pki_dir'], self.ACC) - pre = os.path.join(self.opts['pki_dir'], self.PEND) - rejected = os.path.join(self.opts['pki_dir'], self.REJ) - return accepted, pre, rejected, None - - def check_minion_cache(self, preserve_minions=False): - ''' - Check the minion cache to make sure that old minion data is cleared - ''' - keys = self.list_keys() - minions = [] - for key, val in six.iteritems(keys): - minions.extend(val) - - m_cache = os.path.join(self.opts['cachedir'], 'minions') - if not self.opts.get('preserve_minion_cache', False): - if os.path.isdir(m_cache): - for minion in os.listdir(m_cache): - if minion not in minions and minion not in preserve_minions: - try: - shutil.rmtree(os.path.join(m_cache, minion)) - except (OSError, IOError) as ex: - log.warning('RaetKey: Delete cache for %s got OSError/IOError: %s \n', - minion, - ex) - continue - cache = salt.cache.factory(self.opts) - clist = cache.list(self.ACC) - if clist: - for minion in clist: - if minion not in minions and minion not in preserve_minions: - cache.flush('{0}/{1}'.format(self.ACC, minion)) - - kind = self.opts.get('__role', '') # application kind - if kind not in salt.utils.kinds.APPL_KINDS: - emsg = ("Invalid application kind = '{0}'.".format(kind)) - log.error(emsg + '\n') - raise ValueError(emsg) - role = self.opts.get('id', '') - if not role: - emsg = ("Invalid id.") - log.error(emsg + "\n") - raise ValueError(emsg) - - name = "{0}_{1}".format(role, kind) - road_cache = os.path.join(self.opts['cachedir'], - 'raet', - name, - 'remote') - if os.path.isdir(road_cache): - for road in os.listdir(road_cache): - root, ext = os.path.splitext(road) - if ext not in ('.json', '.msgpack'): - continue - prefix, sep, name = root.partition('.') - if not name or prefix != 'estate': - continue - path = os.path.join(road_cache, road) - with salt.utils.files.fopen(path, 'rb') as fp_: - if ext == '.json': - data = salt.utils.json.load(fp_) - elif ext == '.msgpack': - data = msgpack.load(fp_) - role = salt.utils.stringutils.to_unicode(data['role']) - if role not in minions: - os.remove(path) - - def gen_keys(self, keydir=None, keyname=None, keysize=None, user=None): - ''' - Use libnacl to generate and safely save a private key - ''' - import libnacl.dual # pylint: disable=import-error,3rd-party-module-not-gated - d_key = libnacl.dual.DualSecret() - keydir, keyname, _, _ = self._get_key_attrs(keydir, keyname, - keysize, user) - path = '{0}.key'.format(os.path.join( - keydir, - keyname)) - d_key.save(path, 'msgpack') - - def check_master(self): - ''' - Log if the master is not running - NOT YET IMPLEMENTED - ''' - return True - - def local_keys(self): - ''' - Return a dict of local keys - ''' - ret = {'local': []} - fn_ = os.path.join(self.opts['pki_dir'], 'local.key') - if os.path.isfile(fn_): - ret['local'].append(fn_) - return ret - - def status(self, minion_id, pub, verify): - ''' - Accepts the minion id, device id, curve public and verify keys. - If the key is not present, put it in pending and return "pending", - If the key has been accepted return "accepted" - if the key should be rejected, return "rejected" - ''' - acc, pre, rej, _ = self._check_minions_directories() # pylint: disable=W0632 - acc_path = os.path.join(acc, minion_id) - pre_path = os.path.join(pre, minion_id) - rej_path = os.path.join(rej, minion_id) - # open mode is turned on, force accept the key - pub = salt.utils.stringutils.to_str(pub) - verify = salt.utils.stringutils.to_str(verify) - keydata = { - 'minion_id': minion_id, - 'pub': pub, - 'verify': verify} - if self.opts['open_mode']: # always accept and overwrite - with salt.utils.files.fopen(acc_path, 'w+b') as fp_: - fp_.write(self.serial.dumps(keydata)) - return self.ACC - if os.path.isfile(rej_path): - log.debug("Rejection Reason: Keys already rejected.\n") - return self.REJ - elif os.path.isfile(acc_path): - # The minion id has been accepted, verify the key strings - with salt.utils.files.fopen(acc_path, 'rb') as fp_: - keydata = self.serial.loads(fp_.read()) - if keydata['pub'] == pub and keydata['verify'] == verify: - return self.ACC - else: - log.debug("Rejection Reason: Keys not match prior accepted.\n") - return self.REJ - elif os.path.isfile(pre_path): - auto_reject = self.auto_key.check_autoreject(minion_id) - auto_sign = self.auto_key.check_autosign(minion_id) - with salt.utils.files.fopen(pre_path, 'rb') as fp_: - keydata = self.serial.loads(fp_.read()) - if keydata['pub'] == pub and keydata['verify'] == verify: - if auto_reject: - self.reject(minion_id) - log.debug("Rejection Reason: Auto reject pended.\n") - return self.REJ - elif auto_sign: - self.accept(minion_id) - return self.ACC - return self.PEND - else: - log.debug("Rejection Reason: Keys not match prior pended.\n") - return self.REJ - # This is a new key, evaluate auto accept/reject files and place - # accordingly - auto_reject = self.auto_key.check_autoreject(minion_id) - auto_sign = self.auto_key.check_autosign(minion_id) - if self.opts['auto_accept']: - w_path = acc_path - ret = self.ACC - elif auto_sign: - w_path = acc_path - ret = self.ACC - elif auto_reject: - w_path = rej_path - log.debug("Rejection Reason: Auto reject new.\n") - ret = self.REJ - else: - w_path = pre_path - ret = self.PEND - with salt.utils.files.fopen(w_path, 'w+b') as fp_: - fp_.write(self.serial.dumps(keydata)) - return ret - - def _get_key_str(self, minion_id, status): - ''' - Return the key string in the form of: - - pub: - verify: - ''' - path = os.path.join(self.opts['pki_dir'], status, minion_id) - with salt.utils.files.fopen(path, 'rb') as fp_: - keydata = self.serial.loads(fp_.read()) - return 'pub: {0}\nverify: {1}'.format( - keydata['pub'], - keydata['verify']) - - def _get_key_finger(self, path): - ''' - Return a sha256 kingerprint for the key - ''' - with salt.utils.files.fopen(path, 'rb') as fp_: - keydata = self.serial.loads(fp_.read()) - key = 'pub: {0}\nverify: {1}'.format( - keydata['pub'], - keydata['verify']) - return hashlib.sha256(key).hexdigest() - - def key_str(self, match): - ''' - Return the specified public key or keys based on a glob - ''' - ret = {} - for status, keys in six.iteritems(self.name_match(match)): - ret[status] = {} - for key in salt.utils.data.sorted_ignorecase(keys): - ret[status][key] = self._get_key_str(key, status) - return ret - - def key_str_all(self): - ''' - Return all managed key strings - ''' - ret = {} - for status, keys in six.iteritems(self.list_keys()): - ret[status] = {} - for key in salt.utils.data.sorted_ignorecase(keys): - ret[status][key] = self._get_key_str(key, status) - return ret - - def accept(self, match=None, match_dict=None, include_rejected=False, include_denied=False): - ''' - Accept public keys. If "match" is passed, it is evaluated as a glob. - Pre-gathered matches can also be passed via "match_dict". - ''' - if match is not None: - matches = self.name_match(match) - elif match_dict is not None and isinstance(match_dict, dict): - matches = match_dict - else: - matches = {} - keydirs = [self.PEND] - if include_rejected: - keydirs.append(self.REJ) - if include_denied: - keydirs.append(self.DEN) - for keydir in keydirs: - for key in matches.get(keydir, []): - try: - shutil.move( - os.path.join( - self.opts['pki_dir'], - keydir, - key), - os.path.join( - self.opts['pki_dir'], - self.ACC, - key) - ) - except (IOError, OSError): - pass - return ( - self.name_match(match) if match is not None - else self.dict_match(matches) - ) - - def accept_all(self): - ''' - Accept all keys in pre - ''' - keys = self.list_keys() - for key in keys[self.PEND]: - try: - shutil.move( - os.path.join( - self.opts['pki_dir'], - self.PEND, - key), - os.path.join( - self.opts['pki_dir'], - self.ACC, - key) - ) - except (IOError, OSError): - pass - return self.list_keys() - - def delete_key(self, - match=None, - match_dict=None, - preserve_minions=None, - revoke_auth=False): - ''' - Delete public keys. If "match" is passed, it is evaluated as a glob. - Pre-gathered matches can also be passed via "match_dict". - ''' - if match is not None: - matches = self.name_match(match) - elif match_dict is not None and isinstance(match_dict, dict): - matches = match_dict - else: - matches = {} - for status, keys in six.iteritems(matches): - for key in keys: - if revoke_auth: - if self.opts.get('rotate_aes_key') is False: - print('Immediate auth revocation specified but AES key rotation not allowed. ' - 'Minion will not be disconnected until the master AES key is rotated.') - else: - try: - client = salt.client.get_local_client(mopts=self.opts) - client.cmd_async(key, 'saltutil.revoke_auth') - except salt.exceptions.SaltClientError: - print('Cannot contact Salt master. ' - 'Connection for {0} will remain up until ' - 'master AES key is rotated or auth is revoked ' - 'with \'saltutil.revoke_auth\'.'.format(key)) - try: - os.remove(os.path.join(self.opts['pki_dir'], status, key)) - except (OSError, IOError): - pass - if self.opts.get('preserve_minions') is True: - self.check_minion_cache(preserve_minions=matches.get('minions', [])) - else: - self.check_minion_cache() - return ( - self.name_match(match) if match is not None - else self.dict_match(matches) - ) - - def delete_all(self): - ''' - Delete all keys - ''' - for status, keys in six.iteritems(self.list_keys()): - for key in keys: - try: - os.remove(os.path.join(self.opts['pki_dir'], status, key)) - except (OSError, IOError): - pass - self.check_minion_cache() - return self.list_keys() - - def reject(self, match=None, match_dict=None, include_accepted=False, include_denied=False): - ''' - Reject public keys. If "match" is passed, it is evaluated as a glob. - Pre-gathered matches can also be passed via "match_dict". - ''' - if match is not None: - matches = self.name_match(match) - elif match_dict is not None and isinstance(match_dict, dict): - matches = match_dict - else: - matches = {} - keydirs = [self.PEND] - if include_accepted: - keydirs.append(self.ACC) - if include_denied: - keydirs.append(self.DEN) - for keydir in keydirs: - for key in matches.get(keydir, []): - try: - shutil.move( - os.path.join( - self.opts['pki_dir'], - keydir, - key), - os.path.join( - self.opts['pki_dir'], - self.REJ, - key) - ) - except (IOError, OSError): - pass - self.check_minion_cache() - return ( - self.name_match(match) if match is not None - else self.dict_match(matches) - ) - - def reject_all(self): - ''' - Reject all keys in pre - ''' - keys = self.list_keys() - for key in keys[self.PEND]: - try: - shutil.move( - os.path.join( - self.opts['pki_dir'], - self.PEND, - key), - os.path.join( - self.opts['pki_dir'], - self.REJ, - key) - ) - except (IOError, OSError): - pass - self.check_minion_cache() - return self.list_keys() - - def finger(self, match, hash_type=None): - ''' - Return the fingerprint for a specified key - ''' - if hash_type is None: - hash_type = __opts__['hash_type'] - - matches = self.name_match(match, True) - ret = {} - for status, keys in six.iteritems(matches): - ret[status] = {} - for key in keys: - if status == 'local': - path = os.path.join(self.opts['pki_dir'], key) - else: - path = os.path.join(self.opts['pki_dir'], status, key) - ret[status][key] = self._get_key_finger(path) - return ret - - def finger_all(self, hash_type=None): - ''' - Return fingerprints for all keys - ''' - if hash_type is None: - hash_type = __opts__['hash_type'] - - ret = {} - for status, keys in six.iteritems(self.list_keys()): - ret[status] = {} - for key in keys: - if status == 'local': - path = os.path.join(self.opts['pki_dir'], key) - else: - path = os.path.join(self.opts['pki_dir'], status, key) - ret[status][key] = self._get_key_finger(path) - return ret - - def read_all_remote(self): - ''' - Return a dict of all remote key data - ''' - data = {} - for status, mids in six.iteritems(self.list_keys()): - for mid in mids: - keydata = self.read_remote(mid, status) - if keydata: - keydata['acceptance'] = status - data[mid] = keydata - - return data - - def read_remote(self, minion_id, status=ACC): - ''' - Read in a remote key of status - ''' - path = os.path.join(self.opts['pki_dir'], status, minion_id) - if not os.path.isfile(path): - return {} - with salt.utils.files.fopen(path, 'rb') as fp_: - return self.serial.loads(fp_.read()) - - def read_local(self): - ''' - Read in the local private keys, return an empy dict if the keys do not - exist - ''' - path = os.path.join(self.opts['pki_dir'], 'local.key') - if not os.path.isfile(path): - return {} - with salt.utils.files.fopen(path, 'rb') as fp_: - return self.serial.loads(fp_.read()) - - def write_local(self, priv, sign): - ''' - Write the private key and the signing key to a file on disk - ''' - keydata = {'priv': priv, - 'sign': sign} - path = os.path.join(self.opts['pki_dir'], 'local.key') - with salt.utils.files.set_umask(0o277): - if os.path.exists(path): - #mode = os.stat(path).st_mode - os.chmod(path, stat.S_IWUSR | stat.S_IRUSR) - with salt.utils.files.fopen(path, 'w+') as fp_: - fp_.write(self.serial.dumps(keydata)) - os.chmod(path, stat.S_IRUSR) - - def delete_local(self): - ''' - Delete the local private key file - ''' - path = os.path.join(self.opts['pki_dir'], 'local.key') - if os.path.isfile(path): - os.remove(path) - - def delete_pki_dir(self): - ''' - Delete the private key directory - ''' - path = self.opts['pki_dir'] - if os.path.exists(path): - shutil.rmtree(path) diff --git a/salt/master.py b/salt/master.py index 606be7319cb..abd91377085 100644 --- a/salt/master.py +++ b/salt/master.py @@ -13,7 +13,6 @@ import os import re import sys import time -import errno import signal import stat import logging @@ -2359,58 +2358,3 @@ class ClearFuncs(object): Send the load back to the sender. ''' return clear_load - - -class FloMWorker(MWorker): - ''' - Change the run and bind to be ioflo friendly - ''' - def __init__(self, - opts, - key, - ): - MWorker.__init__(self, opts, key) - - def setup(self): - ''' - Prepare the needed objects and socket for iteration within ioflo - ''' - salt.utils.crypt.appendproctitle(self.__class__.__name__) - self.clear_funcs = salt.master.ClearFuncs( - self.opts, - self.key, - ) - self.aes_funcs = salt.master.AESFuncs(self.opts) - self.context = zmq.Context(1) - self.socket = self.context.socket(zmq.REP) - if self.opts.get('ipc_mode', '') == 'tcp': - self.w_uri = 'tcp://127.0.0.1:{0}'.format( - self.opts.get('tcp_master_workers', 4515) - ) - else: - self.w_uri = 'ipc://{0}'.format( - os.path.join(self.opts['sock_dir'], 'workers.ipc') - ) - log.info('ZMQ Worker binding to socket %s', self.w_uri) - self.poller = zmq.Poller() - self.poller.register(self.socket, zmq.POLLIN) - self.socket.connect(self.w_uri) - - def handle_request(self): - ''' - Handle a single request - ''' - try: - polled = self.poller.poll(1) - if polled: - package = self.socket.recv() - self._update_aes() - payload = self.serial.loads(package) - ret = self.serial.dumps(self._handle_payload(payload)) - self.socket.send(ret) - except KeyboardInterrupt: - raise - except Exception as exc: - # Properly handle EINTR from SIGUSR1 - if isinstance(exc, zmq.ZMQError) and exc.errno == errno.EINTR: - return diff --git a/salt/modules/event.py b/salt/modules/event.py index 32aff4f37f2..f6a043923de 100644 --- a/salt/modules/event.py +++ b/salt/modules/event.py @@ -45,19 +45,6 @@ def fire_master(data, tag, preload=None): # We can't send an event if we're in masterless mode log.warning('Local mode detected. Event with tag %s will NOT be sent.', tag) return False - if __opts__['transport'] == 'raet': - channel = salt.transport.client.ReqChannel.factory(__opts__) - load = {'id': __opts__['id'], - 'tag': tag, - 'data': data, - 'cmd': '_minion_event'} - try: - channel.send(load) - except Exception: - pass - finally: - channel.close() - return True if preload or __opts__.get('__cli') == 'salt-call': # If preload is specified, we must send a raw event (this is diff --git a/salt/modules/minion.py b/salt/modules/minion.py index ec5d2c5a77a..76fd832b2e6 100644 --- a/salt/modules/minion.py +++ b/salt/modules/minion.py @@ -35,20 +35,12 @@ def list_(): salt 'master' minion.list ''' pki_dir = __salt__['config.get']('pki_dir', '') - transport = __salt__['config.get']('transport', '') # We have to replace the minion/master directories pki_dir = pki_dir.replace('minion', 'master') # The source code below is (nearly) a copy of salt.key.Key.list_keys - - # We have to differentiate between RaetKey._check_minions_directories - # and Zeromq-Keys. Raet-Keys only have three states while ZeroMQ-keys - # have an additional 'denied' state. - if transport in ('zeromq', 'tcp'): - key_dirs = _check_minions_directories(pki_dir) - else: - key_dirs = _check_minions_directories_raetkey(pki_dir) + key_dirs = _check_minions_directories(pki_dir) ret = {} @@ -80,19 +72,6 @@ def _check_minions_directories(pki_dir): return minions_accepted, minions_pre, minions_rejected, minions_denied -def _check_minions_directories_raetkey(pki_dir): - ''' - Return the minion keys directory paths. - - This function is a copy of salt.key.RaetKey._check_minions_directories. - ''' - accepted = os.path.join(pki_dir, salt.key.RaetKey.ACC) - pre = os.path.join(pki_dir, salt.key.RaetKey.PEND) - rejected = os.path.join(pki_dir, salt.key.RaetKey.REJ) - - return accepted, pre, rejected - - def kill(timeout=15): ''' Kill the salt minion. diff --git a/salt/modules/raet_publish.py b/salt/modules/raet_publish.py deleted file mode 100644 index d7a6dd3cd43..00000000000 --- a/salt/modules/raet_publish.py +++ /dev/null @@ -1,241 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Publish a command from a minion to a target -''' -from __future__ import absolute_import, unicode_literals, print_function - -# Import python libs -import time -import logging - -# Import salt libs -import salt.payload -import salt.utils.args -import salt.transport.client -from salt.exceptions import SaltReqTimeoutError - -# Import 3rd party libs -from salt.ext import six - -log = logging.getLogger(__name__) - -__virtualname__ = 'publish' - - -def __virtual__(): - return __virtualname__ if __opts__.get('transport', '') == 'raet' else False - - -def _parse_args(arg): - ''' - yamlify `arg` and ensure it's outermost datatype is a list - ''' - yaml_args = salt.utils.args.yamlify_arg(arg) - - if yaml_args is None: - return [] - elif not isinstance(yaml_args, list): - return [yaml_args] - else: - return yaml_args - - -def _publish( - tgt, - fun, - arg=None, - tgt_type='glob', - returner='', - timeout=5, - form='clean'): - ''' - Publish a command from the minion out to other minions, publications need - to be enabled on the Salt master and the minion needs to have permission - to publish the command. The Salt master will also prevent a recursive - publication loop, this means that a minion cannot command another minion - to command another minion as that would create an infinite command loop. - - The arguments sent to the minion publish function are separated with - commas. This means that for a minion executing a command with multiple - args it will look like this:: - - salt system.example.com publish.publish '*' user.add 'foo,1020,1020' - - CLI Example: - - .. code-block:: bash - - salt system.example.com publish.publish '*' cmd.run 'ls -la /tmp' - ''' - if fun == 'publish.publish': - log.info('Function name is \'publish.publish\'. Returning {}') - return {} - - arg = _parse_args(arg) - - load = {'cmd': 'minion_pub', - 'fun': fun, - 'arg': arg, - 'tgt': tgt, - 'tgt_type': tgt_type, - 'ret': returner, - 'tmo': timeout, - 'form': form, - 'id': __opts__['id']} - - channel = salt.transport.client.ReqChannel.factory(__opts__) - try: - try: - peer_data = channel.send(load) - except SaltReqTimeoutError: - return '\'{0}\' publish timed out'.format(fun) - if not peer_data: - return {} - # CLI args are passed as strings, re-cast to keep time.sleep happy - time.sleep(float(timeout)) - load = {'cmd': 'pub_ret', - 'id': __opts__['id'], - 'jid': six.text_type(peer_data['jid'])} - ret = channel.send(load) - if form == 'clean': - cret = {} - for host in ret: - cret[host] = ret[host]['ret'] - return cret - else: - return ret - finally: - channel.close() - - -def publish(tgt, - fun, - arg=None, - tgt_type='glob', - returner='', - timeout=5): - ''' - Publish a command from the minion out to other minions. - - Publications need to be enabled on the Salt master and the minion - needs to have permission to publish the command. The Salt master - will also prevent a recursive publication loop, this means that a - minion cannot command another minion to command another minion as - that would create an infinite command loop. - - The ``tgt_type`` argument is used to pass a target other than a glob into - the execution, the available options are: - - - glob - - pcre - - grain - - grain_pcre - - pillar - - pillar_pcre - - ipcidr - - range - - compound - - .. versionchanged:: 2017.7.0 - The ``expr_form`` argument has been renamed to ``tgt_type``, earlier - releases must use ``expr_form``. - - The arguments sent to the minion publish function are separated with - commas. This means that for a minion executing a command with multiple - args it will look like this: - - .. code-block:: bash - - salt system.example.com publish.publish '*' user.add 'foo,1020,1020' - salt system.example.com publish.publish 'os:Fedora' network.interfaces '' grain - - CLI Example: - - .. code-block:: bash - - salt system.example.com publish.publish '*' cmd.run 'ls -la /tmp' - - - .. admonition:: Attention - - If you need to pass a value to a function argument and that value - contains an equal sign, you **must** include the argument name. - For example: - - .. code-block:: bash - - salt '*' publish.publish test.kwarg arg='cheese=spam' - - - ''' - return _publish(tgt, - fun, - arg=arg, - tgt_type=tgt_type, - returner=returner, - timeout=timeout, - form='clean') - - -def full_data(tgt, - fun, - arg=None, - tgt_type='glob', - returner='', - timeout=5): - ''' - Return the full data about the publication, this is invoked in the same - way as the publish function - - CLI Example: - - .. code-block:: bash - - salt system.example.com publish.full_data '*' cmd.run 'ls -la /tmp' - - .. admonition:: Attention - - If you need to pass a value to a function argument and that value - contains an equal sign, you **must** include the argument name. - For example: - - .. code-block:: bash - - salt '*' publish.full_data test.kwarg arg='cheese=spam' - - ''' - return _publish(tgt, - fun, - arg=arg, - tgt_type=tgt_type, - returner=returner, - timeout=timeout, - form='full') - - -def runner(fun, arg=None, timeout=5): - ''' - Execute a runner on the master and return the data from the runner - function - - CLI Example: - - .. code-block:: bash - - salt publish.runner manage.down - ''' - arg = _parse_args(arg) - - load = {'cmd': 'minion_runner', - 'fun': fun, - 'arg': arg, - 'tmo': timeout, - 'id': __opts__['id']} - - channel = salt.transport.client.ReqChannel.factory(__opts__) - try: - return channel.send(load) - except SaltReqTimeoutError: - return '\'{0}\' runner publish timed out'.format(fun) - finally: - channel.close() diff --git a/salt/runners/manage.py b/salt/runners/manage.py index e89f41328a6..82e0eec275f 100644 --- a/salt/runners/manage.py +++ b/salt/runners/manage.py @@ -20,17 +20,16 @@ from salt.ext import six from salt.ext.six.moves.urllib.request import urlopen as _urlopen # pylint: disable=no-name-in-module,import-error # Import salt libs +import salt.client +import salt.client.ssh import salt.key import salt.utils.compat import salt.utils.files import salt.utils.minions import salt.utils.path -import salt.utils.raetevent -import salt.client -import salt.client.ssh +import salt.utils.versions import salt.wheel import salt.version -from salt.utils.event import tagify from salt.exceptions import SaltClientError, SaltSystemExit FINGERPRINT_REGEX = re.compile(r'^([a-f0-9]{2}:){15}([a-f0-9]{2})$') @@ -218,7 +217,7 @@ def _show_ip_migration(show_ip, show_ipv4): return show_ip -def list_state(subset=None, show_ip=False, show_ipv4=None, state=None): +def list_state(subset=None, show_ip=False, show_ipv4=None): ''' .. versionadded:: 2015.8.0 .. versionchanged:: 2019.2.0 @@ -234,10 +233,6 @@ def list_state(subset=None, show_ip=False, show_ipv4=None, state=None): show_ip : False Also show the IP address each minion is connecting from. - state : 'available' - Show minions being in specific state that is one of 'available', 'joined', - 'allowed', 'alived' or 'reaped'. - CLI Example: .. code-block:: bash @@ -245,30 +240,18 @@ def list_state(subset=None, show_ip=False, show_ipv4=None, state=None): salt-run manage.list_state ''' show_ip = _show_ip_migration(show_ip, show_ipv4) - conf_file = __opts__['conf_file'] - opts = salt.config.client_config(conf_file) - if opts['transport'] == 'raet': - event = salt.utils.raetevent.PresenceEvent(__opts__, __opts__['sock_dir'], state=state) - data = event.get_event(wait=60, tag=tagify('present', 'presence')) - key = 'present' if state is None else state - if not data or key not in data: - minions = [] - else: - minions = data[key] - if subset: - minions = [m for m in minions if m in subset] - else: - # Always return 'present' for 0MQ for now - # TODO: implement other states support for 0MQ - ckminions = salt.utils.minions.CkMinions(__opts__) - minions = ckminions.connected_ids(show_ip=show_ip, subset=subset) + + # Always return 'present' for 0MQ for now + # TODO: implement other states support for 0MQ + ckminions = salt.utils.minions.CkMinions(__opts__) + minions = ckminions.connected_ids(show_ip=show_ip, subset=subset) connected = dict(minions) if show_ip else sorted(minions) return connected -def list_not_state(subset=None, show_ip=False, show_ipv4=None, state=None): +def list_not_state(subset=None, show_ip=False, show_ipv4=None): ''' .. versionadded:: 2015.8.0 .. versionchanged:: 2019.2.0 @@ -284,10 +267,6 @@ def list_not_state(subset=None, show_ip=False, show_ipv4=None, state=None): show_ip : False Also show the IP address each minion is connecting from. - state : 'available' - Show minions being in specific state that is one of 'available', 'joined', - 'allowed', 'alived' or 'reaped'. - CLI Example: .. code-block:: bash @@ -295,20 +274,13 @@ def list_not_state(subset=None, show_ip=False, show_ipv4=None, state=None): salt-run manage.list_not_state ''' show_ip = _show_ip_migration(show_ip, show_ipv4) - connected = list_state(subset=None, show_ip=show_ip, state=state) + connected = list_state(subset=None, show_ip=show_ip) key = salt.key.get_key(__opts__) keys = key.list_keys() - # TODO: Need better way to handle key/node name difference for raet - # In raet case node name is '_' meanwhile the key name - # is just ''. So append '_minion' to the name to match. - appen_kind = isinstance(key, salt.key.RaetKey) - not_connected = [] for minion in keys[key.ACC]: - if appen_kind: - minion += '_minion' if minion not in connected and (subset is None or minion in subset): not_connected.append(minion) @@ -389,7 +361,7 @@ def joined(subset=None, show_ip=False, show_ipv4=None): salt-run manage.joined ''' show_ip = _show_ip_migration(show_ip, show_ipv4) - return list_state(subset=subset, show_ip=show_ip, state='joined') + return list_state(subset=subset, show_ip=show_ip) def not_joined(subset=None, show_ip=False, show_ipv4=None): @@ -415,7 +387,7 @@ def not_joined(subset=None, show_ip=False, show_ipv4=None): salt-run manage.not_joined ''' show_ip = _show_ip_migration(show_ip, show_ipv4) - return list_not_state(subset=subset, show_ip=show_ip, state='joined') + return list_not_state(subset=subset, show_ip=show_ip) def allowed(subset=None, show_ip=False, show_ipv4=None): @@ -441,7 +413,7 @@ def allowed(subset=None, show_ip=False, show_ipv4=None): salt-run manage.allowed ''' show_ip = _show_ip_migration(show_ip, show_ipv4) - return list_state(subset=subset, show_ip=show_ip, state='allowed') + return list_state(subset=subset, show_ip=show_ip) def not_allowed(subset=None, show_ip=False, show_ipv4=None): @@ -467,7 +439,7 @@ def not_allowed(subset=None, show_ip=False, show_ipv4=None): salt-run manage.not_allowed ''' show_ip = _show_ip_migration(show_ip, show_ipv4) - return list_not_state(subset=subset, show_ip=show_ip, state='allowed') + return list_not_state(subset=subset, show_ip=show_ip) def alived(subset=None, show_ip=False, show_ipv4=None): @@ -493,7 +465,7 @@ def alived(subset=None, show_ip=False, show_ipv4=None): salt-run manage.alived ''' show_ip = _show_ip_migration(show_ip, show_ipv4) - return list_state(subset=subset, show_ip=show_ip, state='alived') + return list_state(subset=subset, show_ip=show_ip) def not_alived(subset=None, show_ip=False, show_ipv4=None): @@ -519,7 +491,7 @@ def not_alived(subset=None, show_ip=False, show_ipv4=None): salt-run manage.not_alived ''' show_ip = _show_ip_migration(show_ip, show_ipv4) - return list_not_state(subset=subset, show_ip=show_ip, state='alived') + return list_not_state(subset=subset, show_ip=show_ip) def reaped(subset=None, show_ip=False, show_ipv4=None): @@ -545,7 +517,7 @@ def reaped(subset=None, show_ip=False, show_ipv4=None): salt-run manage.reaped ''' show_ip = _show_ip_migration(show_ip, show_ipv4) - return list_state(subset=subset, show_ip=show_ip, state='reaped') + return list_state(subset=subset, show_ip=show_ip) def not_reaped(subset=None, show_ip=False, show_ipv4=None): @@ -571,69 +543,7 @@ def not_reaped(subset=None, show_ip=False, show_ipv4=None): salt-run manage.not_reaped ''' show_ip = _show_ip_migration(show_ip, show_ipv4) - return list_not_state(subset=subset, show_ip=show_ip, state='reaped') - - -def get_stats(estate=None, stack='road'): - ''' - Print the stack stats - - estate : None - The name of the target estate. Master stats would be requested by default - - stack : 'road' - Show stats on either road or lane stack - Allowed values are 'road' or 'lane'. - - CLI Example: - - .. code-block:: bash - - salt-run manage.get_stats [estate=alpha_minion] [stack=lane] - ''' - conf_file = __opts__['conf_file'] - opts = salt.config.client_config(conf_file) - if opts['transport'] == 'raet': - tag = tagify(stack, 'stats') - event = salt.utils.raetevent.StatsEvent(__opts__, __opts__['sock_dir'], tag=tag, estate=estate) - stats = event.get_event(wait=60, tag=tag) - else: - # TODO: implement 0MQ analog - stats = 'Not implemented' - - return stats - - -def road_stats(estate=None): - ''' - Print the estate road stack stats - - estate : None - The name of the target estate. Master stats would be requested by default - - CLI Example: - - .. code-block:: bash - - salt-run manage.road_stats [estate=alpha_minion] - ''' - return get_stats(estate=estate, stack='road') - - -def lane_stats(estate=None): - ''' - Print the estate manor lane stack stats - - estate : None - The name of the target estate. Master stats would be requested by default - - CLI Example: - - .. code-block:: bash - - salt-run manage.lane_stats [estate=alpha_minion] - ''' - return get_stats(estate=estate, stack='lane') + return list_not_state(subset=subset, show_ip=show_ip) def safe_accept(target, tgt_type='glob'): diff --git a/salt/transport/client.py b/salt/transport/client.py index ca83ac93762..3daf3a5e009 100644 --- a/salt/transport/client.py +++ b/salt/transport/client.py @@ -112,15 +112,12 @@ class AsyncReqChannel(AsyncChannel): AsyncChannel._config_resolver() import salt.transport.tcp return salt.transport.tcp.AsyncTCPReqChannel(opts, **kwargs) - elif ttype == 'raet': - import salt.transport.raet - return salt.transport.raet.RAETReqChannel(opts, **kwargs) elif ttype == 'local': import salt.transport.local return salt.transport.local.AsyncLocalChannel(opts, **kwargs) else: raise Exception( - 'Channels are only defined for tcp, zeromq, raet, and local' + 'Channels are only defined for tcp, zeromq, and local' ) # return NewKindOfChannel(opts, **kwargs) @@ -166,15 +163,12 @@ class AsyncPubChannel(AsyncChannel): AsyncChannel._config_resolver() import salt.transport.tcp return salt.transport.tcp.AsyncTCPPubChannel(opts, **kwargs) - elif ttype == 'raet': # TODO: - import salt.transport.raet - return salt.transport.raet.AsyncRAETPubChannel(opts, **kwargs) elif ttype == 'local': # TODO: import salt.transport.local return salt.transport.local.AsyncLocalPubChannel(opts, **kwargs) else: raise Exception( - 'Channels are only defined for tcp, zeromq, raet, and local' + 'Channels are only defined for tcp, zeromq, and local' ) # return NewKindOfChannel(opts, **kwargs) diff --git a/salt/transport/raet.py b/salt/transport/raet.py deleted file mode 100644 index c4a87f8e179..00000000000 --- a/salt/transport/raet.py +++ /dev/null @@ -1,162 +0,0 @@ -# -*- coding: utf-8 -*- -''' -RAET transport classes -''' - -from __future__ import absolute_import, print_function, unicode_literals -import time - -# Import Salt Libs -import logging - -import salt.utils.kinds as kinds -from salt.transport.client import ReqChannel - -log = logging.getLogger(__name__) - -try: - from raet import raeting, nacling - from raet.lane.stacking import LaneStack - from raet.lane.yarding import RemoteYard -except (ImportError, OSError): - # Don't die on missing transport libs since only one transport is required - pass - -# Module globals for default LaneStack. Because RAETReqChannels are created on demand -# they do not have access to the master estate that motivated their creation -# Also in Raet a LaneStack can be shared shared by all channels in a given jobber -# For these reasons module globals are used to setup a shared jobber_stack as -# well has routing information for the master that motivated the jobber -# when a channel is not used in a jobber context then a LaneStack is created -# on demand. - -jobber_stack = None # module global that holds raet jobber LaneStack -jobber_rxMsgs = {} # dict of deques one for each RAETReqChannel for the jobber -jobber_estate_name = None # module global of motivating master estate name -jobber_yard_name = None # module global of motivating master yard name - - -class RAETReqChannel(ReqChannel): - ''' - Build the communication framework to communicate over the local process - uxd socket and send messages forwarded to the master. then wait for the - relative return message. - - Two use cases: - mininion to master communication, normal use case - Minion is communicating via yard through minion Road to master - The destination route needs the estate name of the associated master - master call via runner, special use case - In the special case the master call external process is communicating - via a yard with the master manor yard - The destination route estate is None to indicate local estate - - The difference between the two is how the destination route - is assigned. - ''' - - def __init__(self, opts, usage=None, **kwargs): - self.opts = opts - self.ttype = 'raet' - if usage == 'master_call': # runner.py master_call - self.dst = (None, None, 'local_cmd') - else: # everything else minion to master including salt-call - self.dst = (jobber_estate_name or None, - jobber_yard_name or None, - 'remote_cmd') - self.stack = None - self.ryn = 'manor' # remote yard name - - def __prep_stack(self): - ''' - Prepare the stack objects - ''' - global jobber_stack - if not self.stack: - if jobber_stack: - self.stack = jobber_stack - else: - self.stack = jobber_stack = self._setup_stack(ryn=self.ryn) - log.debug("RAETReqChannel Using Jobber Stack at = %s\n", self.stack.ha) - - def _setup_stack(self, ryn='manor'): - ''' - Setup and return the LaneStack and Yard used by by channel when global - not already setup such as in salt-call to communicate to-from the minion - - ''' - role = self.opts.get('id') - if not role: - emsg = ("Missing role(\'id\') required to setup RAETReqChannel.") - log.error(emsg + "\n") - raise ValueError(emsg) - - kind = self.opts.get('__role') # application kind 'master', 'minion', etc - if kind not in kinds.APPL_KINDS: - emsg = ("Invalid application kind = '{0}' for RAETReqChannel.".format(kind)) - log.error(emsg + "\n") - raise ValueError(emsg) - if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.master], - kinds.APPL_KIND_NAMES[kinds.applKinds.syndic]]: - lanename = 'master' - elif kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - kinds.APPL_KIND_NAMES[kinds.applKinds.caller]]: - lanename = "{0}_{1}".format(role, kind) - else: - emsg = ("Unsupported application kind '{0}' for RAETReqChannel.".format(kind)) - log.error(emsg + '\n') - raise ValueError(emsg) - - name = 'channel' + nacling.uuid(size=18) - stack = LaneStack(name=name, - lanename=lanename, - sockdirpath=self.opts['sock_dir']) - - stack.Pk = raeting.PackKind.pack - stack.addRemote(RemoteYard(stack=stack, - name=ryn, - lanename=lanename, - dirpath=self.opts['sock_dir'])) - log.debug("Created Channel Jobber Stack %s\n", stack.name) - return stack - - def crypted_transfer_decode_dictentry(self, load, dictkey=None, tries=3, timeout=60): - ''' - We don't need to do the crypted_transfer_decode_dictentry routine for - raet, just wrap send. - ''' - return self.send(load, tries, timeout) - - def send(self, load, tries=3, timeout=60, raw=False): - ''' - Send a message load and wait for a relative reply - One shot wonder - ''' - self.__prep_stack() - tried = 1 - start = time.time() - track = nacling.uuid(18) - src = (None, self.stack.local.name, track) - self.route = {'src': src, 'dst': self.dst} - msg = {'route': self.route, 'load': load} - self.stack.transmit(msg, self.stack.nameRemotes[self.ryn].uid) - while track not in jobber_rxMsgs: - self.stack.serviceAll() - while self.stack.rxMsgs: - msg, sender = self.stack.rxMsgs.popleft() - jobber_rxMsgs[msg['route']['dst'][2]] = msg - continue - if track in jobber_rxMsgs: - break - if time.time() - start > timeout: - if tried >= tries: - raise ValueError("Message send timed out after '{0} * {1}'" - " secs. route = {2} track = {3} load={4}".format(tries, - timeout, - self.route, - track, - load)) - self.stack.transmit(msg, self.stack.nameRemotes['manor'].uid) - tried += 1 - time.sleep(0.01) - return jobber_rxMsgs.pop(track).get('return', {}) diff --git a/salt/transport/server.py b/salt/transport/server.py index 30fe5bcf9d8..ffc937ea355 100644 --- a/salt/transport/server.py +++ b/salt/transport/server.py @@ -31,9 +31,6 @@ class ReqServerChannel(object): if ttype == 'zeromq': import salt.transport.zeromq return salt.transport.zeromq.ZeroMQReqServerChannel(opts) - elif ttype == 'raet': - import salt.transport.raet - return salt.transport.raet.RAETReqServerChannel(opts) elif ttype == 'tcp': import salt.transport.tcp return salt.transport.tcp.TCPReqServerChannel(opts) @@ -41,7 +38,7 @@ class ReqServerChannel(object): import salt.transport.local return salt.transport.local.LocalServerChannel(opts) else: - raise Exception('Channels are only defined for ZeroMQ and raet') + raise Exception('Channels are only defined for ZeroMQ and TCP') # return NewKindOfChannel(opts, **kwargs) def pre_fork(self, process_manager): @@ -79,9 +76,6 @@ class PubServerChannel(object): if ttype == 'zeromq': import salt.transport.zeromq return salt.transport.zeromq.ZeroMQPubServerChannel(opts, **kwargs) - elif ttype == 'raet': # TODO: - import salt.transport.raet - return salt.transport.raet.RAETPubServerChannel(opts, **kwargs) elif ttype == 'tcp': import salt.transport.tcp return salt.transport.tcp.TCPPubServerChannel(opts) @@ -89,7 +83,7 @@ class PubServerChannel(object): import salt.transport.local return salt.transport.local.LocalPubServerChannel(opts, **kwargs) else: - raise Exception('Channels are only defined for ZeroMQ and raet') + raise Exception('Channels are only defined for ZeroMQ and TCP') # return NewKindOfChannel(opts, **kwargs) def pre_fork(self, process_manager, kwargs=None): diff --git a/salt/utils/event.py b/salt/utils/event.py index 0b877980aaa..be82604a99b 100644 --- a/salt/utils/event.py +++ b/salt/utils/event.py @@ -128,27 +128,20 @@ def get_event( ''' sock_dir = sock_dir or opts['sock_dir'] # TODO: AIO core is separate from transport - if transport in ('zeromq', 'tcp'): - if node == 'master': - return MasterEvent(sock_dir, - opts, - listen=listen, - io_loop=io_loop, - keep_loop=keep_loop, - raise_errors=raise_errors) - return SaltEvent(node, - sock_dir, - opts, - listen=listen, - io_loop=io_loop, - keep_loop=keep_loop, - raise_errors=raise_errors) - elif transport == 'raet': - import salt.utils.raetevent - return salt.utils.raetevent.RAETEvent(node, - sock_dir=sock_dir, - listen=listen, - opts=opts) + if node == 'master': + return MasterEvent(sock_dir, + opts, + listen=listen, + io_loop=io_loop, + keep_loop=keep_loop, + raise_errors=raise_errors) + return SaltEvent(node, + sock_dir, + opts, + listen=listen, + io_loop=io_loop, + keep_loop=keep_loop, + raise_errors=raise_errors) def get_master_event(opts, sock_dir, listen=True, io_loop=None, raise_errors=False): @@ -158,11 +151,6 @@ def get_master_event(opts, sock_dir, listen=True, io_loop=None, raise_errors=Fal # TODO: AIO core is separate from transport if opts['transport'] in ('zeromq', 'tcp', 'detect'): return MasterEvent(sock_dir, opts, listen=listen, io_loop=io_loop, raise_errors=raise_errors) - elif opts['transport'] == 'raet': - import salt.utils.raetevent - return salt.utils.raetevent.MasterEvent( - opts=opts, sock_dir=sock_dir, listen=listen - ) def fire_args(opts, jid, tag_data, prefix=''): diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index 5a3c8b1ac0e..4df2c3f6368 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -33,7 +33,6 @@ import salt.utils.args import salt.utils.data import salt.utils.files import salt.utils.jid -import salt.utils.kinds as kinds import salt.utils.platform import salt.utils.process import salt.utils.stringutils @@ -2757,48 +2756,8 @@ class SaltCallOptionParser(six.with_metaclass(OptionParserMeta, else: opts = config.minion_config(self.get_config_file_path(), cache_minion_id=True) - - if opts.get('transport') == 'raet': - if not self._find_raet_minion(opts): # must create caller minion - opts['__role'] = kinds.APPL_KIND_NAMES[kinds.applKinds.caller] return opts - def _find_raet_minion(self, opts): - ''' - Returns true if local RAET Minion is available - ''' - yardname = 'manor' - dirpath = opts['sock_dir'] - - role = opts.get('id') - if not role: - emsg = "Missing role required to setup RAET SaltCaller." - logger.error(emsg) - raise ValueError(emsg) - - kind = opts.get('__role') # application kind 'master', 'minion', etc - if kind not in kinds.APPL_KINDS: - emsg = "Invalid application kind = '{0}' for RAET SaltCaller.".format(six.text_type(kind)) - logger.error(emsg) - raise ValueError(emsg) - - if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.minion], kinds.APPL_KIND_NAMES[kinds.applKinds.caller]]: - lanename = "{0}_{1}".format(role, kind) - else: - emsg = "Unsupported application kind '{0}' for RAET SaltCaller.".format(six.text_type(kind)) - logger.error(emsg) - raise ValueError(emsg) - - if kind == kinds.APPL_KIND_NAMES[kinds.applKinds.minion]: # minion check - try: - from raet.lane.yarding import Yard # pylint: disable=3rd-party-module-not-gated - ha, dirpath = Yard.computeHa(dirpath, lanename, yardname) # pylint: disable=invalid-name - if os.path.exists(ha) and not os.path.isfile(ha) and not os.path.isdir(ha): # minion manor yard - return True - except ImportError as ex: - logger.error("Error while importing Yard: %s", ex) - return False - def process_module_dirs(self): for module_dir in self.options.module_dirs: # Provide some backwards compatibility with previous comma diff --git a/salt/utils/raetevent.py b/salt/utils/raetevent.py deleted file mode 100644 index eb1569b9534..00000000000 --- a/salt/utils/raetevent.py +++ /dev/null @@ -1,340 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Manage events - -This module is used to manage events via RAET -''' -# pylint: disable=3rd-party-module-not-gated - -# Import python libs -from __future__ import absolute_import, print_function, unicode_literals -import os -import logging -import time -from collections import MutableMapping - -# Import salt libs -import salt.payload -import salt.loader -import salt.state -import salt.utils.event -import salt.utils.kinds as kinds -from salt import transport -from salt import syspaths - -try: - from raet import raeting, nacling - from raet.lane.stacking import LaneStack - from raet.lane.yarding import RemoteYard - HAS_RAET = True -except ImportError: - HAS_RAET = False - -# Import 3rd-party libs -from salt.ext import six - -log = logging.getLogger(__name__) - - -def __virtual__(): - return HAS_RAET - - -class RAETEvent(object): - ''' - The base class used to manage salt events - ''' - def __init__(self, node, sock_dir=None, listen=True, opts=None): - ''' - Set up the stack and remote yard - ''' - self.node = node # application kind see kinds.APPL_KIND_NAMES - self.sock_dir = sock_dir - if opts is None: - opts = {} - self.opts = opts - self.stack = None - self.ryn = 'manor' # remote yard name - self.connected = False - self.cpub = False - self.__load_cache_regex() - self.__prep_stack(listen) - - @classmethod - def __load_cache_regex(cls): - ''' - Initialize the regular expression cache and put it in the - class namespace. The regex search strings will be prepend with '^' - ''' - # This is in the class namespace, to minimize cache memory - # usage and maximize cache hits - # The prepend='^' is to reduce differences in behavior between - # the default 'startswith' and the optional 'regex' match_type - cls.cache_regex = salt.utils.cache.CacheRegex(prepend='^') - - def __prep_stack(self, listen): - ''' - Prepare the stack objects - ''' - if not self.stack: - if hasattr(transport, 'jobber_stack') and transport.jobber_stack: - self.stack = transport.jobber_stack - else: - self.stack = transport.jobber_stack = self._setup_stack(ryn=self.ryn) - log.debug("RAETEvent Using Jobber Stack at = %s\n", self.stack.ha) - if listen: - self.subscribe() - - def _setup_stack(self, ryn='manor'): - kind = self.opts.get('__role', '') # opts optional for master - if kind: # not all uses of Raet SaltEvent has opts defined - if kind not in kinds.APPL_KINDS: - emsg = ("Invalid application kind = '{0}' for RAET SaltEvent.".format(kind)) - log.error(emsg + "\n") - raise ValueError(emsg) - if kind != self.node: - emsg = ("Mismatch between node = '{0}' and kind = '{1}' in " - "RAET SaltEvent.".format(self.node, kind)) - log.error(emsg + '\n') - raise ValueError(emsg) - - if self.node in [kinds.APPL_KIND_NAMES[kinds.applKinds.master], - kinds.APPL_KIND_NAMES[kinds.applKinds.syndic]]: # []'master', 'syndic'] - lanename = 'master' - elif self.node in [kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - kinds.APPL_KIND_NAMES[kinds.applKinds.caller]]: # ['minion', 'caller'] - role = self.opts.get('id', '') # opts required for minion - if not role: - emsg = ("Missing role required to setup RAET SaltEvent.") - log.error(emsg + "\n") - raise ValueError(emsg) - if not kind: - emsg = "Missing kind required to setup RAET SaltEvent." - log.error(emsg + '\n') - raise ValueError(emsg) - lanename = "{0}_{1}".format(role, kind) - else: - emsg = ("Unsupported application node kind '{0}' for RAET SaltEvent.".format(self.node)) - log.error(emsg + '\n') - raise ValueError(emsg) - - name = 'event' + nacling.uuid(size=18) - cachedir = self.opts.get('cachedir', os.path.join(syspaths.CACHE_DIR, self.node)) - - stack = LaneStack( - name=name, - lanename=lanename, - sockdirpath=self.sock_dir) - stack.Pk = raeting.PackKind.pack.value - stack.addRemote(RemoteYard(stack=stack, - lanename=lanename, - name=ryn, - dirpath=self.sock_dir)) - return stack - - def subscribe(self, tag=None, match_type=None): - ''' - Included for compat with zeromq events, not required - ''' - if not self.connected: - self.connect_pub() - - def unsubscribe(self, tag=None, match_type=None): - ''' - Included for compat with zeromq events, not required - ''' - return - - def connect_pub(self): - ''' - Establish the publish connection - ''' - try: - route = {'dst': (None, self.ryn, 'event_req'), - 'src': (None, self.stack.local.name, None)} - msg = {'route': route} - self.stack.transmit(msg, self.stack.nameRemotes[self.ryn].uid) - self.stack.serviceAll() - self.connected = True - self.cpub = True - except Exception: - pass - - def connect_pull(self, timeout=1000): - ''' - Included for compat with zeromq events, not required - ''' - return - - @classmethod - def unpack(cls, raw, serial=None): - ''' - Included for compat with zeromq events, not required - ''' - return raw - - def get_event(self, wait=5, tag='', match_type=None, full=False, no_block=None, - auto_reconnect=False): - ''' - Get a single publication. - IF no publication available THEN block for up to wait seconds - AND either return publication OR None IF no publication available. - - IF wait is 0 then block forever. - ''' - if not self.connected: - self.connect_pub() - start = time.time() - while True: - self.stack.serviceAll() - if self.stack.rxMsgs: - msg, sender = self.stack.rxMsgs.popleft() - if 'tag' not in msg and 'data' not in msg: - # Invalid event, how did this get here? - continue - if not msg['tag'].startswith(tag) and self.cache_regex.get(tag).search(msg['tag']) is None: - # Not what we are looking for, throw it away - continue - if full: - return msg - else: - return msg['data'] - if start + wait < time.time(): - return None - time.sleep(0.01) - - def get_event_noblock(self): - ''' - Get the raw event msg without blocking or any other niceties - ''' - if not self.connected: - self.connect_pub() - self.stack.serviceAll() - if self.stack.rxMsgs: - msg, sender = self.stack.rxMsgs.popleft() - if 'tag' not in msg and 'data' not in msg: - # Invalid event, how did this get here? - return None - return msg - - def iter_events(self, tag='', full=False, auto_reconnect=False): - ''' - Creates a generator that continuously listens for events - ''' - while True: - data = self.get_event(tag=tag, full=full, auto_reconnect=auto_reconnect) - if data is None: - continue - yield data - - def fire_event(self, data, tag, timeout=1000): - ''' - Send a single event into the publisher with paylod dict "data" and event - identifier "tag" - ''' - # Timeout is retained for compat with zeromq events - if not six.text_type(tag): # no empty tags allowed - raise ValueError('Empty tag.') - - if not isinstance(data, MutableMapping): # data must be dict - raise ValueError('Dict object expected, not \'{0}\'.'.format(data)) - route = {'dst': (None, self.ryn, 'event_fire'), - 'src': (None, self.stack.local.name, None)} - msg = {'route': route, 'tag': tag, 'data': data} - self.stack.transmit(msg, self.stack.nameRemotes[self.ryn].uid) - self.stack.serviceAll() - - def fire_ret_load(self, load): - ''' - Fire events based on information in the return load - ''' - if load.get('retcode') and load.get('fun'): - # Minion fired a bad retcode, fire an event - if load['fun'] in salt.utils.event.SUB_EVENT: - try: - for tag, data in six.iteritems(load.get('return', {})): - data['retcode'] = load['retcode'] - tags = tag.split('_|-') - if data.get('result') is False: - self.fire_event( - data, - '{0}.{1}'.format(tags[0], tags[-1])) # old dup event - data['jid'] = load['jid'] - data['id'] = load['id'] - data['success'] = False - data['return'] = 'Error: {0}.{1}'.format(tags[0], tags[-1]) - data['fun'] = load['fun'] - data['user'] = load['user'] - self.fire_event( - data, - salt.utils.event.tagify([load['jid'], - 'sub', - load['id'], - 'error', - load['fun']], - 'job')) - except Exception: - pass - - def close_pub(self): - ''' - Here for compatability - ''' - return - - def destroy(self): - if hasattr(self, 'stack'): - self.stack.server.close() - - -class MasterEvent(RAETEvent): - ''' - Create a master event management object - ''' - def __init__(self, opts, sock_dir, listen=True): - super(MasterEvent, self).__init__('master', opts=opts, sock_dir=sock_dir, listen=listen) - - -class PresenceEvent(MasterEvent): - - def __init__(self, opts, sock_dir, listen=True, state=None): - self.state = state - super(PresenceEvent, self).__init__(opts=opts, sock_dir=sock_dir, listen=listen) - - def connect_pub(self): - ''' - Establish the publish connection - ''' - try: - route = {'dst': (None, self.ryn, 'presence_req'), - 'src': (None, self.stack.local.name, None)} - msg = {'route': route} - if self.state: - msg['data'] = {'state': self.state} - self.stack.transmit(msg, self.stack.nameRemotes[self.ryn].uid) - self.stack.serviceAll() - self.connected = True - except Exception: - pass - - -class StatsEvent(MasterEvent): - - def __init__(self, opts, sock_dir, tag, estate=None, listen=True): - super(StatsEvent, self).__init__(opts=opts, sock_dir=sock_dir, listen=listen) - self.tag = tag - self.estate = estate - - def connect_pub(self): - ''' - Establish the publish connection - ''' - try: - route = {'dst': (self.estate, None, 'stats_req'), - 'src': (None, self.stack.local.name, None)} - msg = {'route': route, 'tag': self.tag} - self.stack.transmit(msg, self.stack.nameRemotes[self.ryn].uid) - self.stack.serviceAll() - self.connected = True - except Exception: - pass diff --git a/salt/utils/raetlane.py b/salt/utils/raetlane.py deleted file mode 100644 index 210a3dfb271..00000000000 --- a/salt/utils/raetlane.py +++ /dev/null @@ -1,181 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Provides RAET LaneStack interface for interprocess communications in Salt Raet -to a remote yard, default name for remote is 'manor' . - -Usages are for RAETChannels and RAETEvents -This provides a single module global LaneStack to be shared by all users in -the same process. This combines into one stack the channel and event bus. - -The module attributes: - lane_rx_msgs - is a dict of deques keyed by the destination share name - recipients each value deque holds messages that were addressed - to that share name - - lane_stack - is the shared LaneStack object - - lane_estate_name - is the motivating master estate name when applicable - - lane_yard_name - is the motivating master yard name when applicable - - Because RaetChannels are created on demand - they do not have access to the master estate that motivated their creation - the module globals lane_estate_name and lane_yard_name are provided to setup - so that channels using the routing information for the master that motivated the jobber - when a channel is not used in a jobber context then a LaneStack is created - on demand. - -Example usage: - -import raetlane - -raetlane.prep() - -track = nacling.uuid(18) -src = (None, 'localyardname', track) -dst = ('remotestackname', 'remoteyardname', 'remotesharename') -route = {'src': src, 'dst': dst} -msg = {'route': route, 'body': {}} - -raetlane.transmit(msg) -raetlane.service() - -msg = raetlane.wait(share=track, timeout=5.0) -if not msg: - raise ValueError("Timed out out waiting for response") -''' -# Import python libs -from __future__ import absolute_import, print_function, unicode_literals - -try: - from raet import raeting, nacling - from raet.lane.stacking import LaneStack - from raet.lane.yarding import RemoteYard - HAS_RAET = True -except ImportError: - HAS_RAET = False - -if HAS_RAET: - # pylint: disable=3rd-party-module-not-gated - import time - - # Import Salt Libs - - import logging - import salt.utils.kinds as kinds - - log = logging.getLogger(__name__) - - # Module globals for default shared LaneStack for a process. - rx_msgs = {} # module global dict of deques one for each receipient of msgs - lane_stack = None # module global that holds raet LaneStack - remote_yard = None # module global that holds raet remote Yard - master_estate_name = None # module global of motivating master estate name - master_yard_name = None # module global of motivating master yard name - - def prep(opts, ryn='manor'): - ''' - required items in opts are keys - 'id' - '__role' - 'sock_dir' - - ryn is the remote yard name to communicate with - each use much call raetlane.prep() to ensure lanestack is setup - ''' - if not lane_stack: - _setup(opts=opts, ryn=ryn) - - def _setup(opts, ryn='manor'): - ''' - Setup the LaneStack lane_stack and RemoteYard lane_remote_yard global - ''' - global lane_stack, remote_yard # pylint: disable=W0602 - - role = opts.get('id') - if not role: - emsg = ("Missing role required to setup LaneStack.") - log.error(emsg + "\n") - raise ValueError(emsg) - - kind = opts.get('__role') # application kind 'master', 'minion', etc - if kind not in kinds.APPL_KINDS: - emsg = ("Invalid application kind = '{0}' for LaneStack.".format(kind)) - log.error(emsg + "\n") - raise ValueError(emsg) - if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.master], - kinds.APPL_KIND_NAMES[kinds.applKinds.syndic]]: - lanename = 'master' - elif kind == [kinds.APPL_KIND_NAMES[kinds.applKinds.minion], - kinds.APPL_KIND_NAMES[kinds.applKinds.caller]]: - lanename = "{0}_{1}".format(role, kind) - else: - emsg = ("Unsupported application kind '{0}' for LaneStack.".format(kind)) - log.error(emsg + '\n') - raise ValueError(emsg) - - name = 'lanestack' + nacling.uuid(size=18) - lane_stack = LaneStack(name=name, - lanename=lanename, - sockdirpath=opts['sock_dir']) - - lane_stack.Pk = raeting.PackKind.pack.value - log.debug( - 'Created new LaneStack and local Yard named %s at %s\n', - lane_stack.name, lane_stack.ha - ) - remote_yard = RemoteYard(stack=lane_stack, - name=ryn, - lanename=lanename, - dirpath=opts['sock_dir']) - lane_stack.addRemote(remote_yard) - log.debug( - 'Added to LaneStack %s remote Yard named %s at %s\n', - lane_stack.name, remote_yard.name, remote_yard.ha - ) - - def transmit(msg): - ''' - Sends msg to remote_yard - ''' - lane_stack.transmit(msg, remote_yard.uid) - - def service(): - ''' - Service the lane_stack and move any received messages into their associated - deques in rx_msgs keyed by the destination share in the msg route dict - ''' - lane_stack.serviceAll() - while lane_stack.rxMsgs: - msg, sender = lane_stack.rxMsgs.popleft() - rx_msgs[msg['route']['dst'][2]] = msg - - def receive(share): - ''' - Returns first message from deque at key given by share in rx_msgs if any - otherwise returns None - ''' - service() - if share in rx_msgs: - if rx_msgs[share]: - return rx_msgs[share].popleft() - return None - - def wait(share, timeout=0.0, delay=0.01): - ''' - Blocks until receives a msg addressed to share or timeout - Return msg or None if timed out - Delay is sleep time between services - ''' - start = time.time() - while True: - msg = receive(share) - if msg: - return msg - time.sleep(delay) - if timeout > 0.0 and (time.time() - start) >= timeout: - return None diff --git a/salt/utils/yamldumper.py b/salt/utils/yamldumper.py index 3692ea324aa..c4597f5bb4a 100644 --- a/salt/utils/yamldumper.py +++ b/salt/utils/yamldumper.py @@ -20,13 +20,6 @@ import collections import salt.utils.context from salt.utils.odict import OrderedDict -try: - from ioflo.aid.odicting import odict # pylint: disable=E0611 - HAS_IOFLO = True -except ImportError: - odict = None - HAS_IOFLO = False - __all__ = ['OrderedDumper', 'SafeOrderedDumper', 'IndentedSafeOrderedDumper', 'get_dumper', 'dump', 'safe_dump'] @@ -93,10 +86,6 @@ SafeOrderedDumper.add_representer( 'tag:yaml.org,2002:timestamp', SafeOrderedDumper.represent_scalar) -if HAS_IOFLO: - OrderedDumper.add_representer(odict, represent_ordereddict) - SafeOrderedDumper.add_representer(odict, represent_ordereddict) - def get_dumper(dumper_name): return { diff --git a/salt/version.py b/salt/version.py index 56a870589a2..506a2b89bc2 100644 --- a/salt/version.py +++ b/salt/version.py @@ -583,11 +583,8 @@ def dependency_information(include_salt_cloud=False): ('msgpack-pure', 'msgpack_pure', 'version'), ('pycrypto', 'Crypto', '__version__'), ('pycryptodome', 'Cryptodome', 'version_info'), - ('libnacl', 'libnacl', '__version__'), ('PyYAML', 'yaml', '__version__'), - ('ioflo', 'ioflo', '__version__'), ('PyZMQ', 'zmq', '__version__'), - ('RAET', 'raet', '__version__'), ('ZMQ', 'zmq', 'zmq_version'), ('Mako', 'mako', '__version__'), ('Tornado', 'tornado', 'version'), diff --git a/setup.py b/setup.py index 3e930df8dbd..df66d51e46c 100755 --- a/setup.py +++ b/setup.py @@ -14,7 +14,6 @@ from __future__ import absolute_import, print_function, with_statement import os import sys import glob -import time import operator import platform try: @@ -124,7 +123,6 @@ SALT_VERSION_HARDCODED = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', '_ SALT_SYSPATHS_HARDCODED = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', '_syspaths.py') SALT_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'base.txt') SALT_ZEROMQ_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'zeromq.txt') -SALT_RAET_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'raet.txt') SALT_WINDOWS_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'pkg', 'windows', 'req.txt') SALT_LONG_DESCRIPTION_FILE = os.path.join(os.path.abspath(SETUP_DIRNAME), 'README.rst') @@ -437,7 +435,7 @@ class DownloadWindowsDlls(Command): url = 'https://repo.saltstack.com/windows/dependencies/{bits}/{fname}.dll' dest = os.path.join(os.path.dirname(sys.executable), '{fname}.dll') with indent_log(): - for fname in ('libeay32', 'libsodium', 'ssleay32', 'msvcr120'): + for fname in ('libeay32', 'ssleay32', 'msvcr120'): # See if the library is already on the system if find_library(fname): continue @@ -795,8 +793,9 @@ class SaltDistribution(distutils.dist.Distribution): ''' global_options = distutils.dist.Distribution.global_options + [ ('ssh-packaging', None, 'Run in SSH packaging mode'), - ('salt-transport=', None, 'The transport to prepare salt for. Choices are \'zeromq\' ' - '\'raet\' or \'both\'. Defaults to \'zeromq\'', 'zeromq')] + [ + ('salt-transport=', None, 'The transport to prepare salt for. Currently, the only choice ' + 'is \'zeromq\'. This may be expanded in the future. Defaults to ' + '\'zeromq\'', 'zeromq')] + [ ('with-salt-version=', None, 'Set a fixed version for Salt instead calculating it'), # Salt's Paths Configuration Settings ('salt-root-dir=', None, @@ -1009,19 +1008,11 @@ class SaltDistribution(distutils.dist.Distribution): if self.salt_transport == 'zeromq': install_requires += _parse_requirements_file(SALT_ZEROMQ_REQS) - elif self.salt_transport == 'raet': - install_requires += _parse_requirements_file(SALT_RAET_REQS) if IS_WINDOWS_PLATFORM: install_requires = _parse_requirements_file(SALT_WINDOWS_REQS) return install_requires - @property - def _property_extras_require(self): - if self.ssh_packaging: - return {} - return {'RAET': _parse_requirements_file(SALT_RAET_REQS)} - @property def _property_scripts(self): # Scripts common to all scenarios @@ -1189,15 +1180,7 @@ class SaltDistribution(distutils.dist.Distribution): freezer_includes.append(str(os.path.basename(mod.identifier))) except ImportError: pass - # Include C extension that convinces esky to package up the libsodium C library - # This is needed for ctypes to find it in libnacl which is in turn needed for raet - # see pkg/smartos/esky/sodium_grabber{.c,_installer.py} - freezer_includes.extend([ - 'sodium_grabber', - 'ioflo', - 'raet', - 'libnacl', - ]) + return freezer_includes # <---- Esky Setup ----------------------------------------------------------------------------------------------- @@ -1214,10 +1197,10 @@ class SaltDistribution(distutils.dist.Distribution): elif self.salt_transport is None: self.salt_transport = 'zeromq' - if self.salt_transport not in ('zeromq', 'raet', 'both', 'ssh', 'none'): + if self.salt_transport not in ('zeromq', 'both', 'ssh', 'none'): raise DistutilsArgError( 'The value of --salt-transport needs be \'zeromq\', ' - '\'raet\', \'both\', \'ssh\' or \'none\' not \'{0}\''.format( + '\'both\', \'ssh\', or \'none\' not \'{0}\''.format( self.salt_transport ) ) diff --git a/tests/conftest.py b/tests/conftest.py index 906cca6dd51..e84436edaae 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -88,9 +88,9 @@ def pytest_addoption(parser): parser.addoption( '--transport', default='zeromq', - choices=('zeromq', 'raet', 'tcp'), + choices=('zeromq', 'tcp'), help=('Select which transport to run the integration tests with, ' - 'zeromq, raet, or tcp. Default: %default') + 'zeromq or tcp. Default: %default') ) test_selection_group = parser.getgroup('Tests Selection') test_selection_group.addoption( diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 65009b8c172..2bfd91760db 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -41,9 +41,11 @@ from tests.support.mixins import CheckShellBinaryNameAndVersionMixin, ShellCaseC from tests.support.mixins import AdaptedConfigurationTestCaseMixin, SaltClientTestCaseMixin from tests.support.mixins import SaltMinionEventAssertsMixin, SaltReturnAssertsMixin from tests.support.runtests import RUNTIME_VARS + # Import Salt libs import salt import salt.config +import salt.master import salt.minion import salt.runner import salt.output @@ -58,28 +60,17 @@ import salt.utils.yaml import salt.log.setup as salt_log_setup from salt.utils.verify import verify_env from salt.utils.immutabletypes import freeze -from salt.utils.nb_popen import NonBlockingPopen from salt.exceptions import SaltClientError -try: - import salt.master -except ImportError: - # Not required for raet tests - pass - # Import 3rd-party libs import msgpack from salt.ext import six -from salt.ext.six.moves import cStringIO try: import salt.ext.six.moves.socketserver as socketserver except ImportError: import socketserver -from tornado import gen -from tornado import ioloop - # Import salt tests support libs from tests.support.processes import SaltMaster, SaltMinion, SaltSyndic @@ -213,8 +204,6 @@ class TestDaemon(object): if self.parser.options.transport == 'zeromq': self.start_zeromq_daemons() - elif self.parser.options.transport == 'raet': - self.start_raet_daemons() elif self.parser.options.transport == 'tcp': self.start_tcp_daemons() @@ -506,31 +495,6 @@ class TestDaemon(object): sys.stdout.flush() raise TestDaemonStartFailed() - def start_raet_daemons(self): - ''' - Fire up the raet daemons! - ''' - import salt.daemons.flo - self.master_process = self.start_daemon(salt.daemons.flo.IofloMaster, - self.master_opts, - 'start') - - self.minion_process = self.start_daemon(salt.daemons.flo.IofloMinion, - self.minion_opts, - 'tune_in') - - self.sub_minion_process = self.start_daemon(salt.daemons.flo.IofloMinion, - self.sub_minion_opts, - 'tune_in') - # Wait for the daemons to all spin up - time.sleep(5) - - # self.smaster_process = self.start_daemon(salt.daemons.flo.IofloMaster, - # self.syndic_master_opts, - # 'start') - - # no raet syndic daemon yet - start_tcp_daemons = start_zeromq_daemons def prep_syndic(self): @@ -851,15 +815,6 @@ class TestDaemon(object): proxy_opts['hosts.file'] = os.path.join(TMP, 'rootdir-proxy', 'hosts') proxy_opts['aliases.file'] = os.path.join(TMP, 'rootdir-proxy', 'aliases') - if transport == 'raet': - master_opts['transport'] = 'raet' - master_opts['raet_port'] = 64506 - minion_opts['transport'] = 'raet' - minion_opts['raet_port'] = 64510 - sub_minion_opts['transport'] = 'raet' - sub_minion_opts['raet_port'] = 64520 - # syndic_master_opts['transport'] = 'raet' - if transport == 'tcp': master_opts['transport'] = 'tcp' minion_opts['transport'] = 'tcp' @@ -1047,13 +1002,11 @@ class TestDaemon(object): os.path.join(master_opts['pki_dir'], 'minions_rejected'), os.path.join(master_opts['pki_dir'], 'minions_denied'), os.path.join(master_opts['cachedir'], 'jobs'), - os.path.join(master_opts['cachedir'], 'raet'), os.path.join(master_opts['root_dir'], 'cache', 'tokens'), os.path.join(syndic_master_opts['pki_dir'], 'minions'), os.path.join(syndic_master_opts['pki_dir'], 'minions_pre'), os.path.join(syndic_master_opts['pki_dir'], 'minions_rejected'), os.path.join(syndic_master_opts['cachedir'], 'jobs'), - os.path.join(syndic_master_opts['cachedir'], 'raet'), os.path.join(syndic_master_opts['root_dir'], 'cache', 'tokens'), os.path.join(master_opts['pki_dir'], 'accepted'), os.path.join(master_opts['pki_dir'], 'rejected'), @@ -1061,16 +1014,13 @@ class TestDaemon(object): os.path.join(syndic_master_opts['pki_dir'], 'accepted'), os.path.join(syndic_master_opts['pki_dir'], 'rejected'), os.path.join(syndic_master_opts['pki_dir'], 'pending'), - os.path.join(syndic_master_opts['cachedir'], 'raet'), os.path.join(minion_opts['pki_dir'], 'accepted'), os.path.join(minion_opts['pki_dir'], 'rejected'), os.path.join(minion_opts['pki_dir'], 'pending'), - os.path.join(minion_opts['cachedir'], 'raet'), os.path.join(sub_minion_opts['pki_dir'], 'accepted'), os.path.join(sub_minion_opts['pki_dir'], 'rejected'), os.path.join(sub_minion_opts['pki_dir'], 'pending'), - os.path.join(sub_minion_opts['cachedir'], 'raet'), os.path.dirname(master_opts['log_file']), minion_opts['extension_modules'], sub_minion_opts['extension_modules'], diff --git a/tests/integration/shell/test_key.py b/tests/integration/shell/test_key.py index db7ce7770be..96d806efcb2 100644 --- a/tests/integration/shell/test_key.py +++ b/tests/integration/shell/test_key.py @@ -115,14 +115,6 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin): 'Unaccepted Keys:', 'Rejected Keys:' ] - elif self.master_opts['transport'] == 'raet': - expect = [ - 'Accepted Keys:', - 'minion', - 'sub_minion', - 'Unaccepted Keys:', - 'Rejected Keys:' - ] self.assertEqual(data, expect) def test_list_json_out(self): @@ -143,10 +135,6 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin): 'minions_denied': [], 'minions_pre': [], 'minions': ['minion', 'sub_minion']} - elif self.master_opts['transport'] == 'raet': - expect = {'accepted': ['minion', 'sub_minion'], - 'rejected': [], - 'pending': []} self.assertEqual(ret, expect) def test_list_yaml_out(self): @@ -167,10 +155,6 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin): 'minions_denied': [], 'minions_pre': [], 'minions': ['minion', 'sub_minion']} - elif self.master_opts['transport'] == 'raet': - expect = {'accepted': ['minion', 'sub_minion'], - 'rejected': [], - 'pending': []} self.assertEqual(ret, expect) def test_list_raw_out(self): @@ -193,10 +177,6 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin): 'minions_denied': [], 'minions_pre': [], 'minions': ['minion', 'sub_minion']} - elif self.master_opts['transport'] == 'raet': - expect = {'accepted': ['minion', 'sub_minion'], - 'rejected': [], - 'pending': []} self.assertEqual(ret, expect) def test_list_acc(self): @@ -251,8 +231,6 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin): key_names = None if self.master_opts['transport'] in ('zeromq', 'tcp'): key_names = ('minibar.pub', 'minibar.pem') - elif self.master_opts['transport'] == 'raet': - key_names = ('minibar.key',) for fname in key_names: self.assertTrue(os.path.isfile(os.path.join(tempdir, fname))) finally: diff --git a/tests/integration/shell/test_matcher.py b/tests/integration/shell/test_matcher.py index 917ce38f017..814ab6e7469 100644 --- a/tests/integration/shell/test_matcher.py +++ b/tests/integration/shell/test_matcher.py @@ -229,8 +229,6 @@ class MatchTest(ShellCase, ShellCaseCommonTestsMixin): 'No command was sent, no jid was ' 'assigned.' ) - elif self.master_opts['transport'] == 'raet': - expect = '' self.assertEqual( ''.join(data), expect diff --git a/tests/jenkins.py b/tests/jenkins.py index ed2dd80e09d..42f7f30ca06 100644 --- a/tests/jenkins.py +++ b/tests/jenkins.py @@ -848,9 +848,9 @@ def parse(): parser.add_option( '--test-transport', default='zeromq', - choices=('zeromq', 'raet', 'tcp'), + choices=('zeromq', 'tcp'), help=('Select which transport to run the integration tests with, ' - 'zeromq, raet, or tcp. Default: %default') + 'zeromq or tcp. Default: %default') ) parser.add_option( '--test-without-coverage', diff --git a/tests/minionswarm.py b/tests/minionswarm.py index 1a8e8ac106b..86f3a24a681 100644 --- a/tests/minionswarm.py +++ b/tests/minionswarm.py @@ -166,7 +166,6 @@ class Swarm(object): ''' def __init__(self, opts): self.opts = opts - self.raet_port = 4550 # If given a temp_dir, use it for temporary files if opts['temp_dir']: @@ -329,12 +328,6 @@ class MinionSwarm(Swarm): shutil.copy(minion_pem, minion_pkidir) shutil.copy(minion_pub, minion_pkidir) data['pki_dir'] = minion_pkidir - elif self.opts['transport'] == 'raet': - data['transport'] = 'raet' - data['sock_dir'] = os.path.join(dpath, 'sock') - data['raet_port'] = self.raet_port - data['pki_dir'] = os.path.join(dpath, 'pki') - self.raet_port += 1 elif self.opts['transport'] == 'tcp': data['transport'] = 'tcp' diff --git a/tests/perf/fire_fake.py b/tests/perf/fire_fake.py deleted file mode 100644 index 224bacca8dc..00000000000 --- a/tests/perf/fire_fake.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- encoding: utf-8 -** - -from __future__ import absolute_import, print_function -# Import system libs -import sys -import datetime - -# Import salt libs -import salt.config -import salt.client.raet - - -try: - opts = salt.config.master_config('/etc/salt/master') -except OSError: - print('Could not open master config. Do you need to be root?') - sys.exit(1) - - -class SwarmController(object): - ''' - A controlling class for instantiating and controlling worker procs - ''' - def __init__(self, opts): - self.opts = opts - self.client = salt.client.raet.LocalClient(mopts=opts) - self.total_complete = 0 - - self.run_time = 90 # The number of seconds to run for - self.reqs_sec = 5000 # The number of requests / second to shoot for - self.granularity = 200 # Re-calibrate once for this many runs - self.period_sleep = 0.01 # The number of seconds to initially sleep between pubs - self.ramp_sleep = 0.001 # The number of seconds to ramp up up or down by per calibration - self.start_time = None # The timestamp for the initiation of the test run - - def run(self): - ''' - Run the sequence in a loop - ''' - last_check = 0 - self.start_time = datetime.datetime.now() - goal = self.reqs_sec * self.run_time - while True: - self.fire_it() - last_check += 1 - if last_check > self.granularity: - self.calibrate() - last_check = 0 - if self.total_complete > goal: - print('Test complete') - break - - def fire_it(self): - ''' - Send the pub! - ''' - self.client.pub('silver', 'test.ping') - self.total_complete += 1 - - def calibrate(self): - ''' - Re-calibrate the speed - ''' - elapsed_time = datetime.datetime.now() - self.start_time - #remaining_time = self.run_time - elapsed_time - #remaining_requests = (self.reqs_sec * self.run_time) - self.total_complete - # Figure out what the reqs/sec has been up to this point and then adjust up or down - runtime_reqs_sec = self.total_complete / elapsed_time.total_seconds() - print('Recalibrating. Current reqs/sec: {0}'.format(runtime_reqs_sec)) - return - -controller = SwarmController(opts) -controller.run() diff --git a/tests/runtests.py b/tests/runtests.py index 4a13d8cebdd..2e11af58828 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -261,9 +261,9 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): self.add_option( '--transport', default='zeromq', - choices=('zeromq', 'raet', 'tcp'), + choices=('zeromq', 'tcp'), help=('Select which transport to run the integration tests with, ' - 'zeromq, raet, or tcp. Default: %default') + 'zeromq or tcp. Default: %default') ) self.add_option( '--interactive', diff --git a/tests/support/mixins.py b/tests/support/mixins.py index 30f20bd5f15..45cc6b37ed3 100644 --- a/tests/support/mixins.py +++ b/tests/support/mixins.py @@ -103,7 +103,6 @@ class AdaptedConfigurationTestCaseMixin(object): os.path.join(rdict['pki_dir'], 'minions_rejected'), os.path.join(rdict['pki_dir'], 'minions_denied'), os.path.join(rdict['cachedir'], 'jobs'), - os.path.join(rdict['cachedir'], 'raet'), os.path.join(rdict['cachedir'], 'tokens'), os.path.join(rdict['root_dir'], 'cache', 'tokens'), os.path.join(rdict['pki_dir'], 'accepted'), diff --git a/tests/unit/modules/test_event.py b/tests/unit/modules/test_event.py index e5d2c946ae4..e9140670cb3 100644 --- a/tests/unit/modules/test_event.py +++ b/tests/unit/modules/test_event.py @@ -47,12 +47,6 @@ class EventTestCase(TestCase, LoaderModuleMockMixin): preload = {'id': 'id', 'tag': 'tag', 'data': 'data', 'tok': 'salt', 'cmd': '_minion_event'} - with patch.dict(event.__opts__, {'transport': 'raet', - 'local': False}): - with patch.object(salt_transport_channel_factory, 'send', - return_value=None): - self.assertTrue(event.fire_master('data', 'tag')) - with patch.dict(event.__opts__, {'transport': 'A', 'master_uri': 'localhost', 'local': False}): diff --git a/tests/unit/modules/test_raet_publish.py b/tests/unit/modules/test_raet_publish.py deleted file mode 100644 index 0f04b5e5afa..00000000000 --- a/tests/unit/modules/test_raet_publish.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- -''' - :codeauthor: Jayesh Kariya -''' - -# Import Python libs -from __future__ import absolute_import, print_function, unicode_literals - -# Import Salt Testing Libs -from tests.support.mixins import LoaderModuleMockMixin -from tests.support.unit import skipIf, TestCase -from tests.support.mock import ( - NO_MOCK, - NO_MOCK_REASON, - MagicMock, - patch) - -# Import Salt Libs -import salt.modules.raet_publish as raet_publish -import salt.transport.client -from salt.exceptions import SaltReqTimeoutError - - -@skipIf(NO_MOCK, NO_MOCK_REASON) -class RaetPublishTestCase(TestCase, LoaderModuleMockMixin): - ''' - Test cases for salt.modules.raet_publish - ''' - def setup_loader_modules(self): - return {raet_publish: {}} - - def test_publish(self): - ''' - Test for publish a command from the minion out to other minions. - ''' - with patch.object(raet_publish, '_publish', return_value='A'): - self.assertEqual(raet_publish.publish('tgt', 'fun'), 'A') - - def test_full_data(self): - ''' - Test for return the full data about the publication, - this is invoked in the same way as the publish function - ''' - with patch.object(raet_publish, '_publish', return_value='A'): - self.assertEqual(raet_publish.full_data('tgt', 'fun'), 'A') - - def test_runner(self): - ''' - Test for execute a runner on the master and return - the data from the runner function - ''' - with patch.dict(raet_publish.__opts__, {'id': 'id'}): - with patch.object(salt.transport.client.ReqChannel, 'factory', MagicMock()): - self.assertTrue(raet_publish.runner('fun')) - - class MockFactory(object): - ''' - Mock factory class - ''' - load = '' - - def send(self, load): - ''' - mock send method - ''' - self.load = load - raise SaltReqTimeoutError(load) - - def close(self): - pass - - with patch.dict(raet_publish.__opts__, {'id': 'id'}): - with patch.object(salt.transport.client.ReqChannel, 'factory', - MagicMock(return_value=MockFactory())): - self.assertEqual(raet_publish.runner(1), "'1' runner publish timed out")