saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Remove support for RAET

Browse source 
Conflicts:
* doc/topics/releases/neon.rst
* requirements/tests.txt
* salt/cli/caller.py
* salt/daemons/test/__init__.py
* salt/daemons/test/test_minion.py
* salt/daemons/test/test_saltkeep.py
* salt/modules/event.py
* salt/modules/raet_publish.py
* salt/transport/__init__.py
* salt/utils/parsers.py
* setup.py
* tests/unit/modules/test_raet_publish.py
This commit is contained in:
rallytime 2018-10-05 10:48:14 -04:00 • committed by Pedro Algarvio
parent aee25f6413
commit 4a218142a5
No known key found for this signature in database
GPG key ID: BB36BF6584A298FF
75 changed files with 102 additions and 12908 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
HACKING.rst
View file

@ -155,15 +155,6 @@ ZeroMQ Transport:
pip install -e . pip install -e .
RAET Transport:
.. code-block:: bash
pip install -r requirements/raet.txt
pip install psutil
pip install -e .
Running a self-contained development version Running a self-contained development version
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

24
doc/.tx/config
View file

@ -4112,12 +4112,6 @@ source_file = _build/locale/topics/development/hacking.pot
source_lang = en source_lang = en
source_name = topics/development/hacking.rst source_name = topics/development/hacking.rst
[salt.topics--development--raet--index]
file_filter = locale/<lang>/LC_MESSAGES/topics/development/raet/index.po
source_file = _build/locale/topics/development/raet/index.pot
source_lang = en
source_name = topics/development/raet/index.rst
[salt.topics--development--salt_projects] [salt.topics--development--salt_projects]
file_filter = locale/<lang>/LC_MESSAGES/topics/development/salt_projects.po file_filter = locale/<lang>/LC_MESSAGES/topics/development/salt_projects.po
source_file = _build/locale/topics/development/salt_projects.pot source_file = _build/locale/topics/development/salt_projects.pot
@ -4184,18 +4178,6 @@ source_file = _build/locale/topics/topology/syndic.pot
source_lang = en source_lang = en
source_name = topics/topology/syndic.rst source_name = topics/topology/syndic.rst
[salt.topics--transports--raet--index]
file_filter = locale/<lang>/LC_MESSAGES/topics/transports/raet/index.po
source_file = _build/locale/topics/transports/raet/index.pot
source_lang = en
source_name = topics/transports/raet/index.rst
[salt.topics--transports--raet--programming_intro]
file_filter = locale/<lang>/LC_MESSAGES/topics/transports/raet/programming_intro.po
source_file = _build/locale/topics/transports/raet/programming_intro.pot
source_lang = en
source_name = topics/transports/raet/programming_intro.rst
[salt.topics--tutorials--states_pt5] [salt.topics--tutorials--states_pt5]
file_filter = locale/<lang>/LC_MESSAGES/topics/tutorials/states_pt5.po file_filter = locale/<lang>/LC_MESSAGES/topics/tutorials/states_pt5.po
source_file = _build/locale/topics/tutorials/states_pt5.pot source_file = _build/locale/topics/tutorials/states_pt5.pot
@ -4652,12 +4634,6 @@ source_file = _build/locale/ref/modules/all/salt.modules.pyenv.pot
source_lang = en source_lang = en
source_name = ref/modules/all/salt.modules.pyenv.rst source_name = ref/modules/all/salt.modules.pyenv.rst
[salt.ref--modules--all--salt_modules_raet_publish]
file_filter = locale/<lang>/LC_MESSAGES/ref/modules/all/salt.modules.raet_publish.po
source_file = _build/locale/ref/modules/all/salt.modules.raet_publish.pot
source_lang = en
source_name = ref/modules/all/salt.modules.raet_publish.rst
[salt.ref--modules--all--salt_modules_schedule] [salt.ref--modules--all--salt_modules_schedule]
file_filter = locale/<lang>/LC_MESSAGES/ref/modules/all/salt.modules.schedule.po file_filter = locale/<lang>/LC_MESSAGES/ref/modules/all/salt.modules.schedule.po
source_file = _build/locale/ref/modules/all/salt.modules.schedule.pot source_file = _build/locale/ref/modules/all/salt.modules.schedule.pot

5
doc/ref/configuration/master.rst
View file

@ -866,9 +866,8 @@ Default: ``zeromq``
Changes the underlying transport layer. ZeroMQ is the recommended transport Changes the underlying transport layer. ZeroMQ is the recommended transport
while additional transport layers are under development. Supported values are while additional transport layers are under development. Supported values are
``zeromq``, ``raet`` (experimental), and ``tcp`` (experimental). This setting has ``zeromq`` and ``tcp`` (experimental). This setting has a significant impact on
a significant impact on performance and should not be changed unless you know performance and should not be changed unless you know what you are doing!
what you are doing!
.. code-block:: yaml .. code-block:: yaml

5
doc/ref/configuration/minion.rst
View file

@ -1367,9 +1367,8 @@ Default: ``zeromq``
Changes the underlying transport layer. ZeroMQ is the recommended transport Changes the underlying transport layer. ZeroMQ is the recommended transport
while additional transport layers are under development. Supported values are while additional transport layers are under development. Supported values are
``zeromq``, ``raet`` (experimental), and ``tcp`` (experimental). This setting has ``zeromq`` and ``tcp`` (experimental). This setting has a significant impact
a significant impact on performance and should not be changed unless you know on performance and should not be changed unless you know what you are doing!
what you are doing!
.. code-block:: yaml .. code-block:: yaml

1
doc/ref/modules/all/index.rst
View file

@ -362,7 +362,6 @@ execution modules
qemu_nbd qemu_nbd
quota quota
rabbitmq rabbitmq
raet_publish
rallydev rallydev
random_org random_org
rbac_solaris rbac_solaris

6
doc/ref/modules/all/salt.modules.raet_publish.rst
View file

@ -1,6 +0,0 @@
=========================
salt.modules.raet_publish
=========================
.. automodule:: salt.modules.raet_publish
:members:

1
doc/topics/development/index.rst
View file

@ -10,7 +10,6 @@ Developing Salt
modules/index modules/index
extend/index extend/index
tests/* tests/*
raet/index
git/index git/index
conventions/index conventions/index
../../ref/internals/index ../../ref/internals/index

272
doc/topics/development/raet/index.rst
View file

@ -1,272 +0,0 @@
raet
====
# RAET
# Reliable Asynchronous Event Transport Protocol
.. seealso:: :ref:`RAET Overview <raet>`
Protocol
--------
Layering:
OSI Layers
7: Application: Format: Data (Stack to Application interface buffering etc)
6: Presentation: Format: Data (Encrypt-Decrypt convert to machine independent format)
5: Session: Format: Data (Interhost communications. Authentication. Groups)
4: Transport: Format: Segments (Reliable delivery of Message, Transactions, Segmentation, Error checking)
3: Network: Format: Packets/Datagrams (Addressing Routing)
2: Link: Format: Frames ( Reliable per frame communications connection, Media access controller )
1: Physical: Bits (Transceiver communication connection not reliable)
Link is hidden from Raet
Network is IP host address and Udp Port
Transport is Raet transactions, service kind, tail error checking,
Could include header signing as part of transport reliable delivery serialization of header
Session is session id key exchange for signing. Grouping is Road (like 852 channel)
Presentation is Encrypt Decrypt body Serialize Deserialize Body
Application is body data dictionary
Header signing spans both the Transport and Session layers.
Header
------
JSON Header (Tradeoff some processing speed for extensibility, ease of use, readability)
Body initially JSON but support for "packed" binary body
Packet
------
Header ASCII Safe JSON
Header termination:
Empty line given by double pair of carriage return linefeed
/r/n/r/n
10 13 10 13
ADAD
1010 1101 1010 1101
In json carriage return and newline characters cannot appear in a json encoded
string unless they are escaped with backslash, so the 4 byte combination is illegal in valid
json that does not have multi-byte unicode characters.
These means the header must be ascii safe so no multibyte utf-8 strings
allowed in header.
Following Header Terminator is variable length signature block. This is binary
and the length is provided in the header.
Following the signature block is the packet body or data.
This may either be JSON or packed binary.
The format is given in the json header
Finally is an optional tail block for error checking or encryption details
Header Fields
-------------
In UDP header
sh = source host
sp = source port
dh = destination host
dp = destination port
In RAET Header
hk = header kind
hl = header length
vn = version number
sd = Source Device ID
dd = Destination Device ID
cf = Corresponder Flag
mf = Multicast Flag
si = Session ID
ti = Transaction ID
sk = Service Kind
pk = Packet Kind
bf = Burst Flag (Send all Segments or Ordered packets without interleaved acks)
oi = Order Index
dt = DateTime Stamp
sn = Segment Number
sc = Segment Count
pf = Pending Segment Flag
af = All Flag (Resent all Segments not just one)
nk = Auth header kind
nl = Auth header length
bk = body kind
bl = body length
tk = tail kind
tl = tail length
fg = flags packed (Flags) Default '00' hex string
2 byte Hex string with bits (0, 0, af, pf, 0, bf, mf, cf)
Zeros are TBD flags
Session Bootstrap
-----------------
Minion sends packet with SID of Zero with public key of minions Public Private Key pair
Master acks packet with SID of Zero to let minion know it received the request
Some time later Master sends packet with SID of zero that accepts the Minion
Minion
Session
-------
Session is important for security. Want one session opened and then multiple
transactions within session.
Session ID
SID
sid
GUID hash to guarantee uniqueness since no guarantee of nonvolatile storage
or require file storage to keep last session ID used.
Service Types or Modular Services
---------------------------------
Four Service Types
A) One or more maybe (unacknowledged repeat) maybe means no guarantee
B) Exactly one at most (ack with retries) (duplicate detection idempotent)
at most means fixed number of retries has finite probability of failing
B1) finite retries
B2) infinite retries with exponential back-off up to a maximum delay
C) Exactly one of sequence at most (sequence numbered)
Receiver requests retry of missing packet with same B1 or B2 retry type
D) End to End (Application layer Request Response)
This is two B sub transactions
Initially unicast messaging
Eventually support for Multicast
The use case for C) is to fragment large packets as once a UDP packet
exceeds the frame size its reliability goes way down
So its more reliable to fragment large packets.
Better approach might be to have more modularity.
Services Levels
1) Maybe one or more
A) Fire and forget
no transaction either side
B) Repeat, no ack, no dupdet
repeat counter send side,
no transaction on receive side
C) Repeat, no Ack, dupdet
repeat counter send side,
dup detection transaction receive side
2) More or Less Once
A) retry finite, ack no dupdet
retry timer send side, finite number of retires
ack receive side no dupdet
3) At most Once
A) retry finite, ack, dupdet
retry timer send side, finite number of retires
ack receive side dupdet
4) Exactly once
A) ack retry
retry timer send side,
ack and duplicate detection receive side
Infinite retries with exponential backoff
5) Sequential sequence number
A) reorder escrow
B) Segmented packets
6) request response to application layer
Service Features
1) repeats
2) ack retry transaction id
3) sequence number duplicate detection out of order detection sequencing
4) rep-req
Always include transaction id since multiple transactions on same port
So get duplicate detection for free if keep transaction alive but if use
A) Maybe one or more
B1) At Least One
B2) Exactly One
C) One of sequence
D) End to End
A) Sender creates transaction id for number of repeats but receiver does not
keep transaction alive
B1) Sender creates transaction id keeps it for retries.
Receiver keeps it to send ack then kills so retry could be duplicate not detected
B2) Sender creates transaction id keeps for retries
Receiver keeps tid for acks on any retires so no duplicates.
C) Sender creates TID and Sequence Number.
Receiver checks for out of order sequence and can request retry.
D) Application layer sends response. So question is do we keep transaction open
or have response be new transaction. No because then we need a rep-req ID so
might as well use the same transaction id. Just keep alive until get response.
Little advantage to B1 vs B2 not having duplicates.
So 4 service types
A) Maybe one or more (unacknowledged repeat)
B) Exactly One (At most one) (ack with retry) (duplicate detection idempotent)
C) One of Sequence (sequence numbered)
D) End to End
Also multicast or unicast
Modular Transaction Table
Sender Side:
Transaction ID plus transaction source sender or receiver generated transaction id
Repeat Counter
Retry Timer Retry Counter (finite retries)
Redo Timer (infinite redos with exponential backoff)
Sequence number without acks (look for resend requests)
Sequence with ack (wait for ack before sending next in sequence)
Segmentation
Receiver Side:
Nothing just accept packet
Acknowledge (can delete transaction after acknowledge)
No duplicate detection
Transaction timeout (keep transaction until timeout)
Duplicate detection save transaction id duplicate detection timeout
Request resend of missing packet in sequence
Sequence reordering with escrow timeout wait escrow before requesting resend
Unsegmentation (request resends of missing segment)

21
doc/topics/installation/index.rst
View file

@ -97,27 +97,20 @@ Salt should run on any Unix-like platform so long as the dependencies are met.
* `Tornado`_ - Web framework and asynchronous networking library * `Tornado`_ - Web framework and asynchronous networking library
* `futures`_ - Python2 only dependency. Backport of the concurrent.futures package from Python 3.2 * `futures`_ - Python2 only dependency. Backport of the concurrent.futures package from Python 3.2
Depending on the chosen Salt transport, `ZeroMQ`_ or `RAET`_, dependencies
vary:
* ZeroMQ: * ZeroMQ:
* `ZeroMQ`_ >= 3.2.0 * `ZeroMQ`_ >= 3.2.0
* `pyzmq`_ >= 2.2.0 - ZeroMQ Python bindings * `pyzmq`_ >= 2.2.0 - ZeroMQ Python bindings
* `PyCrypto`_ - The Python cryptography toolkit * `PyCrypto`_ - The Python cryptography toolkit
* RAET:
* `libnacl`_ - Python bindings to `libsodium`_ Salt defaults to the `ZeroMQ`_ transport. The ``--salt-transport`` installation
* `ioflo`_ - The flo programming interface raet and salt-raet is built on option is available, but currently only supports the ``szeromq`` option. This
* `RAET`_ - The worlds most awesome UDP protocol may be expanded in the future.
Salt defaults to the `ZeroMQ`_ transport, and the choice can be made at install
time, for example:
.. code-block:: bash .. code-block:: bash
python setup.py --salt-transport=raet install python setup.py --salt-transport=zeromq install
This way, only the required dependencies are pulled by the setup script if need This way, only the required dependencies are pulled by the setup script if need
be. be.
@ -127,7 +120,7 @@ provided like:
.. code-block:: bash .. code-block:: bash
pip install --install-option="--salt-transport=raet" salt pip install --install-option="--salt-transport=zeromq" salt
.. note:: .. note::
Salt does not bundle dependencies that are typically distributed as part of Salt does not bundle dependencies that are typically distributed as part of
@ -156,10 +149,6 @@ Optional Dependencies
.. _`apache-libcloud`: http://libcloud.apache.org .. _`apache-libcloud`: http://libcloud.apache.org
.. _`Requests`: http://docs.python-requests.org/en/latest .. _`Requests`: http://docs.python-requests.org/en/latest
.. _`Tornado`: http://www.tornadoweb.org/en/stable/ .. _`Tornado`: http://www.tornadoweb.org/en/stable/
.. _`libnacl`: https://github.com/saltstack/libnacl
.. _`ioflo`: https://github.com/ioflo/ioflo
.. _`RAET`: https://github.com/saltstack/raet
.. _`libsodium`: https://github.com/jedisct1/libsodium
.. _`futures`: https://github.com/agronholm/pythonfutures .. _`futures`: https://github.com/agronholm/pythonfutures

1
doc/topics/transports/index.rst
View file

@ -34,4 +34,3 @@ guarantee minion-master confidentiality.
zeromq zeromq
tcp tcp
raet/index

145
doc/topics/transports/raet/index.rst
View file

@ -1,145 +0,0 @@
.. _raet:
==================
The RAET Transport
==================
.. note::
The RAET transport is in very early development, it is functional but no
promises are yet made as to its reliability or security.
As for reliability and security, the encryption used has been audited and
our tests show that raet is reliable. With this said we are still conducting
more security audits and pushing the reliability.
This document outlines the encryption used in RAET
.. versionadded:: 2014.7.0
The Reliable Asynchronous Event Transport, or RAET, is an alternative transport
medium developed specifically with Salt in mind. It has been developed to
allow queuing to happen up on the application layer and comes with socket
layer encryption. It also abstracts a great deal of control over the socket
layer and makes it easy to bubble up errors and exceptions.
RAET also offers very powerful message routing capabilities, allowing for
messages to be routed between processes on a single machine all the way up to
processes on multiple machines. Messages can also be restricted, allowing
processes to be sent messages of specific types from specific sources
allowing for trust to be established.
Using RAET in Salt
==================
Using RAET in Salt is easy, the main difference is that the core dependencies
change, instead of needing pycrypto, M2Crypto, ZeroMQ, and PYZMQ, the packages
`libsodium`_, libnacl, ioflo, and raet are required. Encryption is handled very cleanly
by libnacl, while the queueing and flow control is handled by
ioflo. Distribution packages are forthcoming, but `libsodium`_ can be easily
installed from source, or many distributions do ship packages for it.
The libnacl and ioflo packages can be easily installed from pypi, distribution
packages are in the works.
Once the new deps are installed the 2014.7 release or higher of Salt needs to
be installed.
Once installed, modify the configuration files for the minion and master to
set the transport to raet:
``/etc/salt/master``:
.. code-block:: yaml
transport: raet
``/etc/salt/minion``:
.. code-block:: yaml
transport: raet
Now start salt as it would normally be started, the minion will connect to the
master and share long term keys, which can then in turn be managed via
salt-key. Remote execution and salt states will function in the same way as
with Salt over ZeroMQ.
Limitations
===========
The 2014.7 release of RAET is not complete! The Syndic and Multi Master have
not been completed yet and these are slated for completion in the 2015.5.0
release.
Also, Salt-Raet allows for more control over the client but these hooks have
not been implemented yet, thereforre the client still uses the same system
as the ZeroMQ client. This means that the extra reliability that RAET exposes
has not yet been implemented in the CLI client.
Why?
====
Customer and User Request
-------------------------
Why make an alternative transport for Salt? There are many reasons, but the
primary motivation came from customer requests, many large companies came with
requests to run Salt over an alternative transport, the reasoning was varied,
from performance and scaling improvements to licensing concerns. These
customers have partnered with SaltStack to make RAET a reality.
More Capabilities
-----------------
RAET has been designed to allow salt to have greater communication
capabilities. It has been designed to allow for development into features
which out ZeroMQ topologies can't match.
Many of the proposed features are still under development and will be
announced as they enter proof of concept phases, but these features include
`salt-fuse` - a filesystem over salt, `salt-vt` - a parallel api driven shell
over the salt transport and many others.
RAET Reliability
================
RAET is reliable, hence the name (Reliable Asynchronous Event Transport).
The concern posed by some over RAET reliability is based on the fact that
RAET uses UDP instead of TCP and UDP does not have built in reliability.
RAET itself implements the needed reliability layers that are not natively
present in UDP, this allows RAET to dynamically optimize packet delivery
in a way that keeps it both reliable and asynchronous.
RAET and ZeroMQ
===============
When using RAET, ZeroMQ is not required. RAET is a complete networking
replacement. It is noteworthy that RAET is not a ZeroMQ replacement in a
general sense, the ZeroMQ constructs are not reproduced in RAET, but they are
instead implemented in such a way that is specific to Salt's needs.
RAET is primarily an async communication layer over truly async connections,
defaulting to UDP. ZeroMQ is over TCP and abstracts async constructs within the
socket layer.
Salt is not dropping ZeroMQ support and has no immediate plans to do so.
Encryption
==========
RAET uses Dan Bernstein's NACL encryption libraries and `CurveCP`_ handshake.
The libnacl python binding binds to both `libsodium`_ and tweetnacl to execute
the underlying cryptography. This allows us to completely rely on an
externally developed cryptography system.
Programming Intro
=================
.. toctree::
programming_intro
.. _libsodium: http://doc.libsodium.org/
.. _CurveCP: http://curvecp.org/

41
doc/topics/transports/raet/programming_intro.rst
View file

@ -1,41 +0,0 @@
.. _raet-programming:
=========================
Intro to RAET Programming
=========================
.. note::
This page is still under construction
The first thing to cover is that RAET does not present a socket api, it
presents, and queueing api, all messages in RAET are made available to via
queues. This is the single most differentiating factor with RAET vs other
networking libraries, instead of making a socket, a stack is created.
Instead of calling send() or recv(), messages are placed on the stack to be
sent and messages that are received appear on the stack.
Different kinds of stacks are also available, currently two stacks exist,
the UDP stack, and the UXD stack. The UDP stack is used to communicate over
udp sockets, and the UXD stack is used to communicate over Unix Domain
Sockets.
The UDP stack runs a context for communicating over networks, while the
UXD stack has contexts for communicating between processes.
UDP Stack Messages
==================
To create a UDP stack in RAET, simply create the stack, manage the queues,
and process messages:
.. code-block:: python
from salt.transport.road.raet import stacking
from salt.transport.road.raet import estating
udp_stack = stacking.StackUdp(ha=('127.0.0.1', 7870))
r_estate = estating.Estate(stack=stack, name='foo', ha=('192.168.42.42', 7870))
msg = {'hello': 'world'}
udp_stack.transmit(msg, udp_stack.estates[r_estate.name])
udp_stack.serviceAll()

18
salt/auth/__init__.py
View file

@ -687,18 +687,12 @@ class Resolver(object):
self.auth = salt.loader.auth(opts) self.auth = salt.loader.auth(opts)
def _send_token_request(self, load): def _send_token_request(self, load):
if self.opts['transport'] in ('zeromq', 'tcp'): master_uri = 'tcp://' + salt.utils.zeromq.ip_bracket(self.opts['interface']) + \
master_uri = 'tcp://' + salt.utils.zeromq.ip_bracket(self.opts['interface']) + \ ':' + six.text_type(self.opts['ret_port'])
':' + six.text_type(self.opts['ret_port']) channel = salt.transport.client.ReqChannel.factory(self.opts,
channel = salt.transport.client.ReqChannel.factory(self.opts, crypt='clear',
crypt='clear', master_uri=master_uri)
master_uri=master_uri) return channel.send(load)
return channel.send(load)
elif self.opts['transport'] == 'raet':
channel = salt.transport.client.ReqChannel.factory(self.opts)
channel.dst = (None, None, 'local_cmd')
return channel.send(load)
def cli(self, eauth): def cli(self, eauth):
''' '''

202
salt/cli/caller.py
View file

@ -9,7 +9,6 @@ from __future__ import absolute_import, print_function, unicode_literals
import os import os
import sys import sys
import time
import logging import logging
import traceback import traceback
@ -24,27 +23,11 @@ import salt.transport.client
import salt.utils.args import salt.utils.args
import salt.utils.files import salt.utils.files
import salt.utils.jid import salt.utils.jid
import salt.utils.kinds as kinds
import salt.utils.minion import salt.utils.minion
import salt.utils.profile import salt.utils.profile
import salt.utils.stringutils import salt.utils.stringutils
import salt.defaults.exitcodes import salt.defaults.exitcodes
from salt.cli import daemons
from salt.log import LOG_LEVELS from salt.log import LOG_LEVELS
from salt.utils.platform import is_windows
from salt.utils.process import MultiprocessingProcess
try:
from raet import raeting, nacling
from raet.lane.stacking import LaneStack
from raet.lane.yarding import RemoteYard, Yard
if is_windows():
import win32file
except ImportError:
# Don't die on missing transport libs since only one transport is required
pass
# Import 3rd-party libs # Import 3rd-party libs
from salt.ext import six from salt.ext import six
@ -78,10 +61,8 @@ class Caller(object):
# switch on available ttypes # switch on available ttypes
if ttype in ('zeromq', 'tcp', 'detect'): if ttype in ('zeromq', 'tcp', 'detect'):
return ZeroMQCaller(opts, **kwargs) return ZeroMQCaller(opts, **kwargs)
elif ttype == 'raet':
return RAETCaller(opts, **kwargs)
else: else:
raise Exception('Callers are only defined for ZeroMQ and raet') raise Exception('Callers are only defined for ZeroMQ and TCP')
# return NewKindOfCaller(opts, **kwargs) # return NewKindOfCaller(opts, **kwargs)
@ -317,7 +298,6 @@ class BaseCaller(object):
# Local job cache has been enabled # Local job cache has been enabled
salt.utils.minion.cache_jobs(self.opts, ret['jid'], ret) salt.utils.minion.cache_jobs(self.opts, ret['jid'], ret)
# close raet channel here
return ret return ret
@ -343,183 +323,3 @@ class ZeroMQCaller(BaseCaller):
channel.send(load) channel.send(load)
finally: finally:
channel.close() channel.close()
def raet_minion_run(cleanup_protecteds):
'''
Set up the minion caller. Should be run in its own process.
This function is intentionally left out of RAETCaller. This will avoid
needing to pickle the RAETCaller object on Windows.
'''
minion = daemons.Minion() # daemonizes here
minion.call(cleanup_protecteds=cleanup_protecteds) # caller minion.call_in uses caller.flo
class RAETCaller(BaseCaller):
'''
Object to wrap the calling of local salt modules for the salt-call command
when transport is raet
There are two operation modes.
1) Use a preexisting minion
2) Set up a special caller minion if no preexisting minion
The special caller minion is a subset whose only function is to perform
Salt-calls with raet as the transport
The essentials:
A RoadStack whose local estate name is of the form "role_kind" where:
role is the minion id opts['id']
kind is opts['__role'] which should be 'caller' APPL_KIND_NAMES
The RoadStack if for communication to/from a master
A LaneStack with manor yard so that RaetChannels created by the func Jobbers
can communicate through this manor yard then through the
RoadStack to/from a master
A Router to route between the stacks (Road and Lane)
These are all managed via a FloScript named caller.flo
'''
def __init__(self, opts):
'''
Pass in the command line options
'''
self.process = None
if not opts['local']:
self.stack = self._setup_caller_stack(opts)
salt.transport.jobber_stack = self.stack
if (opts.get('__role') ==
kinds.APPL_KIND_NAMES[kinds.applKinds.caller]):
# spin up and fork minion here
self.process = MultiprocessingProcess(target=raet_minion_run,
kwargs={'cleanup_protecteds': [self.stack.ha], })
self.process.start()
# wait here until '/var/run/salt/minion/alpha_caller.manor.uxd' exists
self._wait_caller(opts)
super(RAETCaller, self).__init__(opts)
def run(self):
'''
Execute the salt call logic
'''
try:
ret = self.call()
if not self.opts['local']:
self.stack.server.close()
salt.transport.jobber_stack = None
if self.opts['print_metadata']:
print_ret = ret
else:
print_ret = ret.get('return', {})
if self.process:
self.process.terminate()
salt.output.display_output(
{'local': print_ret},
out=ret.get('out', 'nested'),
opts=self.opts,
_retcode=ret.get('retcode', salt.defaults.exitcodes.EX_OK))
# _retcode will be available in the kwargs of the outputter function
if self.opts.get('retcode_passthrough', False):
sys.exit(ret['retcode'])
elif ret['retcode'] != salt.defaults.exitcodes.EX_OK:
sys.exit(salt.defaults.exitcodes.EX_GENERIC)
except SaltInvocationError as err:
raise SystemExit(err)
def _setup_caller_stack(self, opts):
'''
Setup and return the LaneStack and Yard used by by channel when global
not already setup such as in salt-call to communicate to-from the minion
'''
role = opts.get('id')
if not role:
emsg = ("Missing role required to setup RAETChannel.")
log.error(emsg + "\n")
raise ValueError(emsg)
kind = opts.get('__role') # application kind 'master', 'minion', etc
if kind not in kinds.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for RAETChannel.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.minion],
kinds.APPL_KIND_NAMES[kinds.applKinds.caller], ]:
lanename = "{0}_{1}".format(role, kind)
else:
emsg = ("Unsupported application kind '{0}' for RAETChannel.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
sockdirpath = opts['sock_dir']
stackname = 'caller' + nacling.uuid(size=18)
stack = LaneStack(name=stackname,
lanename=lanename,
sockdirpath=sockdirpath)
stack.Pk = raeting.PackKind.pack.value
stack.addRemote(RemoteYard(stack=stack,
name='manor',
lanename=lanename,
dirpath=sockdirpath))
log.debug("Created Caller Jobber Stack %s\n", stack.name)
return stack
def _wait_caller(self, opts):
'''
Returns when RAET Minion Yard is available
'''
yardname = 'manor'
dirpath = opts['sock_dir']
role = opts.get('id')
if not role:
emsg = ("Missing role required to setup RAET SaltCaller.")
log.error(emsg + "\n")
raise ValueError(emsg)
kind = opts.get('__role') # application kind 'master', 'minion', etc
if kind not in kinds.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for RAET SaltCaller.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.minion],
kinds.APPL_KIND_NAMES[kinds.applKinds.caller], ]:
lanename = "{0}_{1}".format(role, kind)
else:
emsg = ("Unsupported application kind '{0}' for RAET SaltCaller.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
ha, dirpath = Yard.computeHa(dirpath, lanename, yardname)
if is_windows():
# RAET lanes do not use files on Windows. Need to use win32file
# API to check for existence.
exists = False
while not exists:
try:
f = win32file.CreateFile(
ha,
win32file.GENERIC_WRITE | win32file.GENERIC_READ,
win32file.FILE_SHARE_READ,
None,
win32file.OPEN_EXISTING,
0,
None)
win32file.CloseHandle(f)
exists = True
except win32file.error:
time.sleep(0.1)
else:
while not ((os.path.exists(ha) and
not os.path.isfile(ha) and
not os.path.isdir(ha))):
time.sleep(0.1)
time.sleep(0.5)

83
salt/cli/daemons.py
View file

@ -119,7 +119,7 @@ class Master(salt.utils.parsers.MasterOptionParser, DaemonsMixin): # pylint: di
Creates a master server Creates a master server
''' '''
def _handle_signals(self, signum, sigframe): # pylint: disable=unused-argument def _handle_signals(self, signum, sigframe): # pylint: disable=unused-argument
if hasattr(self.master, 'process_manager'): # IofloMaster has no process manager if hasattr(self.master, 'process_manager'):
# escalate signal to the process manager processes # escalate signal to the process manager processes
self.master.process_manager.stop_restarting() self.master.process_manager.stop_restarting()
self.master.process_manager.send_signal_to_processes(signum) self.master.process_manager.send_signal_to_processes(signum)
@ -156,11 +156,6 @@ class Master(salt.utils.parsers.MasterOptionParser, DaemonsMixin): # pylint: di
self.config['syndic_dir'], self.config['syndic_dir'],
self.config['sqlite_queue_dir'], self.config['sqlite_queue_dir'],
] ]
if self.config.get('transport') == 'raet':
v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted'))
v_dirs.append(os.path.join(self.config['pki_dir'], 'pending'))
v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected'))
v_dirs.append(os.path.join(self.config['cachedir'], 'raet'))
verify_env( verify_env(
v_dirs, v_dirs,
self.config['user'], self.config['user'],
@ -179,21 +174,17 @@ class Master(salt.utils.parsers.MasterOptionParser, DaemonsMixin): # pylint: di
self.action_log_info('Setting up') self.action_log_info('Setting up')
# TODO: AIO core is separate from transport # TODO: AIO core is separate from transport
if self.config['transport'].lower() in ('zeromq', 'tcp'): if not verify_socket(self.config['interface'],
if not verify_socket(self.config['interface'], self.config['publish_port'],
self.config['publish_port'], self.config['ret_port']):
self.config['ret_port']): self.shutdown(4, 'The ports are not available to bind')
self.shutdown(4, 'The ports are not available to bind') self.config['interface'] = ip_bracket(self.config['interface'])
self.config['interface'] = ip_bracket(self.config['interface']) migrations.migrate_paths(self.config)
migrations.migrate_paths(self.config)
# Late import so logging works correctly
import salt.master
self.master = salt.master.Master(self.config)
# Late import so logging works correctly
import salt.master
self.master = salt.master.Master(self.config)
else:
# Add a udp port check here
import salt.daemons.flo
self.master = salt.daemons.flo.IofloMaster(self.config)
self.daemonize_if_required() self.daemonize_if_required()
self.set_pidfile() self.set_pidfile()
salt.utils.process.notify_systemd() salt.utils.process.notify_systemd()
@ -277,12 +268,6 @@ class Minion(salt.utils.parsers.MinionOptionParser, DaemonsMixin): # pylint: di
confd, confd,
] ]
if self.config.get('transport') == 'raet':
v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted'))
v_dirs.append(os.path.join(self.config['pki_dir'], 'pending'))
v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected'))
v_dirs.append(os.path.join(self.config['cachedir'], 'raet'))
verify_env( verify_env(
v_dirs, v_dirs,
self.config['user'], self.config['user'],
@ -318,15 +303,10 @@ class Minion(salt.utils.parsers.MinionOptionParser, DaemonsMixin): # pylint: di
if self.config.get('master_type') == 'func': if self.config.get('master_type') == 'func':
salt.minion.eval_master_func(self.config) salt.minion.eval_master_func(self.config)
self.minion = salt.minion.MinionManager(self.config) self.minion = salt.minion.MinionManager(self.config)
elif transport == 'raet':
import salt.daemons.flo
self.daemonize_if_required()
self.set_pidfile()
self.minion = salt.daemons.flo.IofloMinion(self.config)
else: else:
log.error( log.error(
'The transport \'%s\' is not supported. Please use one of ' 'The transport \'%s\' is not supported. Please use one of '
'the following: tcp, raet, or zeromq.', transport 'the following: tcp, zeromq, or detect.', transport
) )
self.shutdown(1) self.shutdown(1)
@ -384,7 +364,6 @@ class Minion(salt.utils.parsers.MinionOptionParser, DaemonsMixin): # pylint: di
self.prepare() self.prepare()
if check_user(self.config['user']): if check_user(self.config['user']):
self.minion.opts['__role'] = kinds.APPL_KIND_NAMES[kinds.applKinds.caller] self.minion.opts['__role'] = kinds.APPL_KIND_NAMES[kinds.applKinds.caller]
self.minion.opts['raet_cleanup_protecteds'] = cleanup_protecteds
self.minion.call_in() self.minion.call_in()
except (KeyboardInterrupt, SaltSystemExit) as exc: except (KeyboardInterrupt, SaltSystemExit) as exc:
self.action_log_info('Stopping') self.action_log_info('Stopping')
@ -469,12 +448,6 @@ class ProxyMinion(salt.utils.parsers.ProxyMinionOptionParser, DaemonsMixin): #
confd, confd,
] ]
if self.config.get('transport') == 'raet':
v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted'))
v_dirs.append(os.path.join(self.config['pki_dir'], 'pending'))
v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected'))
v_dirs.append(os.path.join(self.config['cachedir'], 'raet'))
verify_env( verify_env(
v_dirs, v_dirs,
self.config['user'], self.config['user'],
@ -497,24 +470,17 @@ class ProxyMinion(salt.utils.parsers.ProxyMinionOptionParser, DaemonsMixin): #
self.shutdown(1) self.shutdown(1)
# TODO: AIO core is separate from transport # TODO: AIO core is separate from transport
if self.config['transport'].lower() in ('zeromq', 'tcp', 'detect'): # Late import so logging works correctly
# Late import so logging works correctly import salt.minion
import salt.minion # If the minion key has not been accepted, then Salt enters a loop
# If the minion key has not been accepted, then Salt enters a loop # waiting for it, if we daemonize later then the minion could halt
# waiting for it, if we daemonize later then the minion could halt # the boot process waiting for a key to be accepted on the master.
# the boot process waiting for a key to be accepted on the master. # This is the latest safe place to daemonize
# This is the latest safe place to daemonize self.daemonize_if_required()
self.daemonize_if_required() self.set_pidfile()
self.set_pidfile() if self.config.get('master_type') == 'func':
if self.config.get('master_type') == 'func': salt.minion.eval_master_func(self.config)
salt.minion.eval_master_func(self.config) self.minion = salt.minion.ProxyMinionManager(self.config)
self.minion = salt.minion.ProxyMinionManager(self.config)
else:
# For proxy minions, this doesn't work yet.
import salt.daemons.flo
self.daemonize_if_required()
self.set_pidfile()
self.minion = salt.daemons.flo.IofloMinion(self.config)
def start(self): def start(self):
''' '''
@ -543,9 +509,6 @@ class ProxyMinion(salt.utils.parsers.ProxyMinionOptionParser, DaemonsMixin): #
log.error(exc) log.error(exc)
self.shutdown(exc.code) self.shutdown(exc.code)
# def call(self, cleanup_protecteds):
# This fn is omitted here, proxy minions have never supported RAET
def shutdown(self, exitcode=0, exitmsg=None): def shutdown(self, exitcode=0, exitmsg=None):
''' '''
If sub-classed, run any shutdown operations on this method. If sub-classed, run any shutdown operations on this method.

8
salt/cli/key.py
View file

@ -17,16 +17,10 @@ class SaltKey(salt.utils.parsers.SaltKeyOptionParser):
''' '''
import salt.key import salt.key
self.parse_args() self.parse_args()
multi = False
if self.config.get('zmq_behavior') and self.config.get('transport') == 'raet':
multi = True
self.setup_logfile_logger() self.setup_logfile_logger()
verify_log(self.config) verify_log(self.config)
if multi: key = salt.key.KeyCLI(self.config)
key = salt.key.MultiKeyCLI(self.config)
else:
key = salt.key.KeyCLI(self.config)
if check_user(self.config['user']): if check_user(self.config['user']):
key.run() key.run()

15
salt/client/__init__.py
View file

@ -97,16 +97,13 @@ def get_local_client(
# Late import to prevent circular import # Late import to prevent circular import
import salt.config import salt.config
opts = salt.config.client_config(c_path) opts = salt.config.client_config(c_path)
if opts['transport'] == 'raet':
import salt.client.raet
return salt.client.raet.LocalClient(mopts=opts)
# TODO: AIO core is separate from transport # TODO: AIO core is separate from transport
elif opts['transport'] in ('zeromq', 'tcp'): return LocalClient(
return LocalClient( mopts=opts,
mopts=opts, skip_perm_errors=skip_perm_errors,
skip_perm_errors=skip_perm_errors, io_loop=io_loop,
io_loop=io_loop, auto_reconnect=auto_reconnect)
auto_reconnect=auto_reconnect)
class LocalClient(object): class LocalClient(object):

102
salt/client/raet/__init__.py
View file

@ -1,102 +0,0 @@
# -*- coding: utf-8 -*-
'''
The client libs to communicate with the salt master when running raet
'''
from __future__ import absolute_import, print_function, unicode_literals
# Import python libs
import os
import time
import logging
# Import Salt libs
import salt.config
import salt.client
import salt.utils.kinds as kinds
import salt.syspaths as syspaths
try:
from raet import raeting, nacling
from raet.lane.stacking import LaneStack
from raet.lane.yarding import RemoteYard
HAS_RAET_LIBS = True
except ImportError:
HAS_RAET_LIBS = False
log = logging.getLogger(__name__)
class LocalClient(salt.client.LocalClient):
'''
The RAET LocalClient
'''
def __init__(self,
c_path=os.path.join(syspaths.CONFIG_DIR, 'master'),
mopts=None):
salt.client.LocalClient.__init__(self, c_path, mopts)
def pub(self,
tgt,
fun,
arg=(),
tgt_type='glob',
ret='',
jid='',
timeout=5,
**kwargs):
'''
Publish the command!
'''
payload_kwargs = self._prep_pub(
tgt,
fun,
arg=arg,
tgt_type=tgt_type,
ret=ret,
jid=jid,
timeout=timeout,
**kwargs)
kind = self.opts['__role']
if kind not in kinds.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for Raet LocalClient.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.master],
kinds.APPL_KIND_NAMES[kinds.applKinds.syndic]]:
lanename = 'master'
else:
emsg = ("Unsupported application kind '{0}' for Raet LocalClient.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
sockdirpath = self.opts['sock_dir']
name = 'client' + nacling.uuid(size=18)
stack = LaneStack(
name=name,
lanename=lanename,
sockdirpath=sockdirpath)
stack.Pk = raeting.PackKind.pack.value
manor_yard = RemoteYard(
stack=stack,
lanename=lanename,
name='manor',
dirpath=sockdirpath)
stack.addRemote(manor_yard)
route = {'dst': (None, manor_yard.name, 'local_cmd'),
'src': (None, stack.local.name, None)}
msg = {'route': route, 'load': payload_kwargs}
stack.transmit(msg)
stack.serviceAll()
while True:
time.sleep(0.01)
stack.serviceAll()
while stack.rxMsgs:
msg, sender = stack.rxMsgs.popleft()
ret = msg.get('return', {})
if 'ret' in ret:
stack.server.close()
return ret['ret']
stack.server.close()
return ret

71
salt/config/__init__.py
View file

@ -960,7 +960,7 @@ VALID_OPTS = {
# The size of key that should be generated when creating new keys # The size of key that should be generated when creating new keys
'keysize': int, 'keysize': int,
# The transport system for this daemon. (i.e. zeromq, raet, etc) # The transport system for this daemon. (i.e. zeromq, tcp, detect, etc)
'transport': six.string_types, 'transport': six.string_types,
# The number of seconds to wait when the client is requesting information about running jobs # The number of seconds to wait when the client is requesting information about running jobs
@ -1010,31 +1010,8 @@ VALID_OPTS = {
'ssh_config_file': six.string_types, 'ssh_config_file': six.string_types,
'ssh_merge_pillar': bool, 'ssh_merge_pillar': bool,
# Enable ioflo verbose logging. Warning! Very verbose!
'ioflo_verbose': int,
'ioflo_period': float,
# Set ioflo to realtime. Useful only for testing/debugging to simulate many ioflo periods very
# quickly
'ioflo_realtime': bool,
# Location for ioflo logs
'ioflo_console_logdir': six.string_types,
# The port to bind to when bringing up a RAET daemon
'raet_port': int,
'raet_alt_port': int,
'raet_mutable': bool,
'raet_main': bool,
'raet_clear_remotes': bool,
'raet_clear_remote_masters': bool,
'raet_road_bufcnt': int,
'raet_lane_bufcnt': int,
'cluster_mode': bool, 'cluster_mode': bool,
'cluster_masters': list,
'sqlite_queue_dir': six.string_types, 'sqlite_queue_dir': six.string_types,
'queue_dirs': list, 'queue_dirs': list,
# Instructs the minion to ping its master(s) every n number of minutes. Used # Instructs the minion to ping its master(s) every n number of minutes. Used
@ -1065,9 +1042,6 @@ VALID_OPTS = {
# Can be set to override the python_shell=False default in the cmd module # Can be set to override the python_shell=False default in the cmd module
'cmd_safe': bool, 'cmd_safe': bool,
# Used strictly for performance testing in RAET.
'dummy_publisher': bool,
# Used by salt-api for master requests timeout # Used by salt-api for master requests timeout
'rest_timeout': int, 'rest_timeout': int,
@ -1473,22 +1447,7 @@ DEFAULT_MINION_OPTS = {
'master_tops_first': False, 'master_tops_first': False,
'auth_safemode': False, 'auth_safemode': False,
'random_master': False, 'random_master': False,
'minion_floscript': os.path.join(FLO_DIR, 'minion.flo'),
'caller_floscript': os.path.join(FLO_DIR, 'caller.flo'),
'ioflo_verbose': 0,
'ioflo_period': 0.1,
'ioflo_realtime': True,
'ioflo_console_logdir': '',
'raet_port': 4510,
'raet_alt_port': 4511,
'raet_mutable': False,
'raet_main': False,
'raet_clear_remotes': True,
'raet_clear_remote_masters': True,
'raet_road_bufcnt': 2,
'raet_lane_bufcnt': 100,
'cluster_mode': False, 'cluster_mode': False,
'cluster_masters': [],
'restart_on_error': False, 'restart_on_error': False,
'ping_interval': 0, 'ping_interval': 0,
'username': None, 'username': None,
@ -1811,23 +1770,7 @@ DEFAULT_MASTER_OPTS = {
'ssh_identities_only': False, 'ssh_identities_only': False,
'ssh_log_file': os.path.join(salt.syspaths.LOGS_DIR, 'ssh'), 'ssh_log_file': os.path.join(salt.syspaths.LOGS_DIR, 'ssh'),
'ssh_config_file': os.path.join(salt.syspaths.HOME_DIR, '.ssh', 'config'), 'ssh_config_file': os.path.join(salt.syspaths.HOME_DIR, '.ssh', 'config'),
'master_floscript': os.path.join(FLO_DIR, 'master.flo'),
'worker_floscript': os.path.join(FLO_DIR, 'worker.flo'),
'maintenance_floscript': os.path.join(FLO_DIR, 'maint.flo'),
'ioflo_verbose': 0,
'ioflo_period': 0.01,
'ioflo_realtime': True,
'ioflo_console_logdir': '',
'raet_port': 4506,
'raet_alt_port': 4511,
'raet_mutable': False,
'raet_main': True,
'raet_clear_remotes': False,
'raet_clear_remote_masters': True,
'raet_road_bufcnt': 2,
'raet_lane_bufcnt': 100,
'cluster_mode': False, 'cluster_mode': False,
'cluster_masters': [],
'sqlite_queue_dir': os.path.join(salt.syspaths.CACHE_DIR, 'master', 'queues'), 'sqlite_queue_dir': os.path.join(salt.syspaths.CACHE_DIR, 'master', 'queues'),
'queue_dirs': [], 'queue_dirs': [],
'cli_summary': False, 'cli_summary': False,
@ -2125,16 +2068,6 @@ def _validate_opts(opts):
if isinstance(opts.get('return'), list): if isinstance(opts.get('return'), list):
opts['return'] = ','.join(opts['return']) opts['return'] = ','.join(opts['return'])
# RAET on Windows uses 'win32file.CreateMailslot()' for IPC. Due to this,
# sock_dirs must start with '\\.\mailslot\' and not contain any colons.
# We don't expect the user to know this, so we will fix up their path for
# them if it isn't compliant.
if (salt.utils.platform.is_windows() and opts.get('transport') == 'raet' and
'sock_dir' in opts and
not opts['sock_dir'].startswith('\\\\.\\mailslot\\')):
opts['sock_dir'] = (
'\\\\.\\mailslot\\' + opts['sock_dir'].replace(':', ''))
for error in errors: for error in errors:
log.warning(error) log.warning(error)
if errors: if errors:
@ -3933,8 +3866,6 @@ def master_config(path, env_var='SALT_MASTER_CONFIG', defaults=None, exit_on_con
opts['nodegroups'] = DEFAULT_MASTER_OPTS.get('nodegroups', {}) opts['nodegroups'] = DEFAULT_MASTER_OPTS.get('nodegroups', {})
if salt.utils.data.is_dictlist(opts['nodegroups']): if salt.utils.data.is_dictlist(opts['nodegroups']):
opts['nodegroups'] = salt.utils.data.repack_dictlist(opts['nodegroups']) opts['nodegroups'] = salt.utils.data.repack_dictlist(opts['nodegroups'])
if opts.get('transport') == 'raet' and 'aes' in opts:
opts.pop('aes')
apply_sdb(opts) apply_sdb(opts)
return opts return opts

8
salt/daemons/__init__.py
View file

@ -12,12 +12,9 @@ try:
except ImportError: except ImportError:
from collections import Iterable, Sequence, Mapping from collections import Iterable, Sequence, Mapping
from collections import namedtuple
import logging import logging
# Import Salt Libs # Import Salt Libs
from salt.utils.odict import OrderedDict
from salt.ext import six from salt.ext import six
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -54,11 +51,6 @@ def extract_masters(opts, masters='master', port=None, raise_if_empty=True):
By default looks for list of masters in opts['master'] and uses By default looks for list of masters in opts['master'] and uses
opts['master_port'] as the default port when otherwise not provided. opts['master_port'] as the default port when otherwise not provided.
To use this function to generate the cluster master list then
call with masters='cluster_masters' and port='raet_port' on a master
and
call with masters='cluster_masters' on a minion
Use the opts key given by masters for the masters list, default is 'master' Use the opts key given by masters for the masters list, default is 'master'
If parameter port is not None then uses the default port given by port If parameter port is not None then uses the default port given by port

195
salt/daemons/flo/__init__.py
View file

@ -1,195 +0,0 @@
# -*- coding: utf-8 -*-
'''
Package for ioflo and raet based daemons and associated ioflo behaviors
To use set
opts['transport'] ='raet'
master minion config
transport: raet
See salt.config.py for relevant defaults
opts['raet_port']
opts['master_floscript']
opts['minion_floscript']
opts['ioflo_period']
opts['ioflo_realtime']
opts['ioflo_verbose']
opts['caller_floscript']
'''
# Import Python libs
from __future__ import absolute_import, print_function, unicode_literals
import os
# Import modules
from . import core
from . import worker
from . import maint
from . import reactor
from . import zero
from . import jobber
from . import dummy
__all__ = ['core', 'worker', 'maint', 'zero', 'dummy', 'jobber', 'reactor']
# Import salt libs
import salt.daemons.masterapi
import salt.utils.stringutils
from salt.ext import six
# Import 3rd-party libs
import ioflo.app.run # pylint: disable=3rd-party-module-not-gated
def explode_opts(opts):
'''
Explode the opts into a preloads list
'''
preloads = [(salt.utils.stringutils.to_str('.salt.opts'), dict(value=opts))]
for key, val in six.iteritems(opts):
ukey = key.replace('.', '_')
preloads.append((salt.utils.stringutils.to_str('.salt.etc.{0}'.format(ukey)), dict(value=val)))
preloads.append((salt.utils.stringutils.to_str('.salt.etc.id'), dict(value=opts['id'])))
return preloads
def warn_deprecated():
salt.utils.versions.warn_until(
'Neon',
'The \'raet\' transport has been deprecated and will be removed in '
'Salt {version}. Please use \'zeromq\' or \'tcp\' transport instead.'
)
class IofloMaster(object):
'''
IofloMaster Class
'''
def __init__(self, opts):
'''
Assign self.opts
'''
warn_deprecated()
self.opts = opts
self.preloads = explode_opts(self.opts)
self.access_keys = salt.daemons.masterapi.access_keys(self.opts)
self.preloads.append(
(salt.utils.stringutils.to_str('.salt.access_keys'), dict(value=self.access_keys)))
def start(self, behaviors=None):
'''
Start up ioflo
port = self.opts['raet_port']
'''
if behaviors is None:
behaviors = []
behaviors.extend(['salt.daemons.flo'])
console_logdir = self.opts.get('ioflo_console_logdir', '')
if console_logdir:
consolepath = os.path.join(console_logdir, 'master.log')
else: # empty means log to std out
consolepath = ''
ioflo.app.run.start(
name='master',
period=float(self.opts['ioflo_period']),
stamp=0.0,
real=self.opts['ioflo_realtime'],
filepath=self.opts['master_floscript'],
behaviors=behaviors,
username="",
password="",
mode=None,
houses=None,
metas=None,
preloads=self.preloads,
verbose=int(self.opts['ioflo_verbose']),
consolepath=consolepath,
)
class IofloMinion(object):
'''
IofloMinion Class
'''
def __init__(self, opts):
'''
Assign self.opts
'''
warn_deprecated()
self.opts = opts
self.restart = False
def tune_in(self, behaviors=None):
'''
Start up ioflo
port = self.opts['raet_port']
'''
if behaviors is None:
behaviors = []
behaviors.extend(['salt.daemons.flo'])
preloads = explode_opts(self.opts)
console_logdir = self.opts.get('ioflo_console_logdir', '')
if console_logdir:
consolepath = os.path.join(console_logdir, 'minion.log')
else: # empty means log to std out
consolepath = ''
ioflo.app.run.start(
name=self.opts['id'],
period=float(self.opts['ioflo_period']),
stamp=0.0,
real=self.opts['ioflo_realtime'],
filepath=self.opts['minion_floscript'],
behaviors=behaviors,
username="",
password="",
mode=None,
houses=None,
metas=None,
preloads=preloads,
verbose=int(self.opts['ioflo_verbose']),
consolepath=consolepath,
)
start = tune_in # alias
def call_in(self, behaviors=None):
'''
Start up caller minion for salt-call when there is no local minion
'''
if behaviors is None:
behaviors = []
behaviors.extend(['salt.daemons.flo'])
preloads = explode_opts(self.opts)
console_logdir = self.opts.get('ioflo_console_logdir', '')
if console_logdir:
consolepath = os.path.join(console_logdir, 'caller.log')
else: # empty means log to std out
consolepath = ''
ioflo.app.run.start(
name=self.opts['id'],
period=float(self.opts['ioflo_period']),
stamp=0.0,
real=self.opts['ioflo_realtime'],
filepath=self.opts['caller_floscript'],
behaviors=behaviors,
username="",
password="",
mode=None,
houses=None,
metas=None,
preloads=preloads,
verbose=int(self.opts['ioflo_verbose']),
consolepath=consolepath,
)

123
salt/daemons/flo/caller.flo
View file

@ -1,123 +0,0 @@
# Salt Caller floscript
house caller
framer callerroadstack be active first setup
# Begin the pre-flight checks
frame setup
enter
do salt raet cleanup
do salt raet road stack setup per inode ".salt.road.manor"
do salt raet manor lane setup
go loadmodules
# Load the minion mods
frame loadmodules
do salt load modules at enter
go start
# OK, let's start the minion up
frame start
# Start the inbound framer
bid start inbound
# Start the bootstrap framer
bid start bootstrap
# Start the eventing framer
bid start eventing
# Start the outbound framer
bid start outbound
# Cleanup on exit
exit
do salt raet road stack closer per inode ".salt.road.manor."
do salt raet lane stack closer per inode ".salt.lane.manor."
framer inbound be inactive first start
frame start
do salt raet road stack service rx
do salt raet lane stack service rx
framer bootstrap be inactive first setup
frame setup
enter
do salt raet road clustered per inode ".salt.road.manor."
do salt raet road usher minion setup per inode ".salt.road.manor."
go clustermaster
go multimaster
frame clustermaster
let if salt.road.manor.cluster.clustered
print Setting Up Master Cluster ....
go join
frame multimaster
print Setting Up Master or MultiMaster
go join
frame join
print Joining...
enter
do salt raet road stack joiner per inode ".salt.road.manor."
recur
do salt raet road stack joined per inode ".salt.road.manor."
do salt raet road stack rejected per inode ".salt.road.manor."
go next if joined in .salt.road.manor.status
go abort if rejected in .salt.road.manor.status
frame joined
print Joined
go next if elapsed >= 0.5
frame allow
print Allowing...
enter
do salt raet road stack allower per inode ".salt.road.manor."
recur
do salt raet road stack allowed per inode ".salt.road.manor."
go next if allowed in .salt.road.manor.status
#go abort if elapsed >= 5
frame allowed
print Allowed
go next if elapsed >= 0.5
frame clustering
print Cluster Setup ...
do salt raet road cluster load setup
go next
frame manager
# start the manager framer
bid start manager #start alive presence from minion side
go next if elapsed >= 3.0
frame loading
print Loading
enter
do salt load modules
go router
frame router
do salt raet router minion
frame abort
bid stop all
framer eventing be inactive first event
frame event
do salt raet eventer
framer manager be inactive first start at 10.0
frame start
do salt raet road stack manager per inode ".salt.road.manor"
framer outbound be inactive first start
frame start
do salt raet lane stack service tx
do salt raet road stack service tx

1791
salt/daemons/flo/core.py
View file

File diff suppressed because it is too large Load diff

69
salt/daemons/flo/dummy.py
View file

@ -1,69 +0,0 @@
# -*- coding: utf-8 -*-
'''
The dummy publisher for the Salt Master
Contains functionality to short-circuit a salt-master's
publish functionality so that instead of publications being
sent across the wire, they are instead transparently redirected
to a returner.
Designed for use primarily in load-testing the salt-master
without the need for a swarm of real minions.
'''
# pylint: disable=W0232
# pylint: disable=3rd-party-module-not-gated
# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
import logging
# Import salt libs
import ioflo.base.deeding
import salt.utils.stringutils
log = logging.getLogger(__name__)
class SaltDummyPublisher(ioflo.base.deeding.Deed):
'''
A dummy publisher that transparently redirects publications to
a translation system to have them mocked up and sent back into a router
'''
Ioinits = {
'opts': salt.utils.stringutils.to_str('.salt.opts'),
'publish': salt.utils.stringutils.to_str('.salt.var.publish'),
'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'),
'workers': salt.utils.stringutils.to_str('.salt.track.workers'),
}
def action(self):
while self.publish.value:
pub = self.publish.value.popleft()
log.debug('Dummy publisher publishing: %s', pub)
msg = self._fill_tmpl(pub)
self.lane_stack.value.transmit(msg, self.lane_stack.value.fetchUidByName(next(self.workers.value)))
def _fill_tmpl(self, pub):
'''
Takes a template and a job and fills the template with
fake return data associated with the job
'''
msg = {'load': {
'fun_args': [],
'jid': pub['return']['pub']['jid'],
'return': True,
'retcode': 0,
'success': True,
'cmd': '_return',
'fun': 'test.ping',
'id': 'silver'
},
'route': {
'src': ('silver_minion', 'jobber50e73ccefd052167c7', 'jid_ret'),
'dst': ('silver_master_master', None, 'remote_cmd')
}
}
log.debug('Dummy publisher faking return with: %s', msg)
return msg

398
salt/daemons/flo/jobber.py
View file

@ -1,398 +0,0 @@
# -*- coding: utf-8 -*-
'''
Jobber Behaviors
'''
# pylint: disable=W0232
# pylint: disable=3rd-party-module-not-gated
# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
import os
import sys
import types
import logging
import traceback
import multiprocessing
import subprocess
# Import salt libs
from salt.ext import six
import salt.daemons.masterapi
import salt.utils.args
import salt.utils.data
import salt.utils.files
import salt.utils.json
import salt.utils.kinds as kinds
import salt.utils.process
import salt.utils.stringutils
import salt.transport
from raet import raeting, nacling
from raet.lane.stacking import LaneStack
from raet.lane.yarding import RemoteYard
from salt.utils.platform import is_windows
from salt.utils.event import tagify
from salt.exceptions import (
CommandExecutionError, CommandNotFoundError, SaltInvocationError)
# Import ioflo libs
import ioflo.base.deeding
from ioflo.base.consoling import getConsole
console = getConsole()
log = logging.getLogger(__name__)
@ioflo.base.deeding.deedify(
salt.utils.stringutils.to_str('SaltRaetShellJobberCheck'),
ioinits={'opts': salt.utils.stringutils.to_str('.salt.opts'),
'grains': salt.utils.stringutils.to_str('.salt.grains'),
'fun': salt.utils.stringutils.to_str('.salt.var.fun'),
'matcher': salt.utils.stringutils.to_str('.salt.matcher'),
'shells': salt.utils.stringutils.to_str('.salt.var.shells'),
'stack': salt.utils.stringutils.to_str('.salt.road.manor.stack')})
def jobber_check(self):
'''
Iterate over the shell jobbers and return the ones that have finished
'''
rms = []
for jid in self.shells.value:
if isinstance(self.shells.value[jid]['proc'].poll(), int):
rms.append(jid)
data = self.shells.value[jid]
stdout, stderr = data['proc'].communicate()
ret = salt.utils.json.loads(
stdout,
object_hook=salt.utils.data.encode_dict if six.PY2 else None
)['local']
route = {'src': (self.stack.value.local.name, 'manor', 'jid_ret'),
'dst': (data['msg']['route']['src'][0], None, 'remote_cmd')}
ret['cmd'] = '_return'
ret['id'] = self.opts.value['id']
ret['jid'] = jid
msg = {'route': route, 'load': ret}
master = self.stack.value.nameRemotes.get(data['msg']['route']['src'][0])
self.stack.value.message(
msg,
master.uid)
for rm_ in rms:
self.shells.value.pop(rm_)
@ioflo.base.deeding.deedify(
salt.utils.stringutils.to_str('SaltRaetShellJobber'),
ioinits={'opts': salt.utils.stringutils.to_str('.salt.opts'),
'grains': salt.utils.stringutils.to_str('.salt.grains'),
'fun': salt.utils.stringutils.to_str('.salt.var.fun'),
'matcher': salt.utils.stringutils.to_str('.salt.matcher'),
'modules': salt.utils.stringutils.to_str('.salt.loader.modules'),
'shells': {'ipath': salt.utils.stringutils.to_str('.salt.var.shells'),
'ival': {}}})
def shell_jobber(self):
'''
Shell jobber start!
'''
while self.fun.value:
msg = self.fun.value.popleft()
data = msg.get('pub')
match = getattr(
self.matcher.value,
'{0}_match'.format(
data.get('tgt_type', 'glob')
)
)(data['tgt'])
if not match:
continue
fun = data['fun']
if fun in self.modules.value:
func = self.modules.value[fun]
else:
continue
args, kwargs = salt.minion.load_args_and_kwargs(
func,
salt.utils.args.parse_input(
data['arg'],
no_parse=data.get('no_parse', [])),
data)
cmd = ['salt-call',
'--out', 'json',
'--metadata',
'-c', salt.syspaths.CONFIG_DIR]
if 'return' in data:
cmd.append('--return')
cmd.append(data['return'])
cmd.append(fun)
for arg in args:
cmd.append(arg)
for key in kwargs:
cmd.append('{0}={1}'.format(key, kwargs[key]))
que = {'pub': data,
'msg': msg}
que['proc'] = subprocess.Popen(
cmd,
shell=False,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
self.shells.value[data['jid']] = que
class SaltRaetNixJobber(ioflo.base.deeding.Deed):
'''
Execute a function call job on a minion on a *nix based system
FloScript:
do salt raet nix jobber
'''
Ioinits = {'opts_store': salt.utils.stringutils.to_str('.salt.opts'),
'grains': salt.utils.stringutils.to_str('.salt.grains'),
'modules': salt.utils.stringutils.to_str('.salt.loader.modules'),
'returners': salt.utils.stringutils.to_str('.salt.loader.returners'),
'module_executors': salt.utils.stringutils.to_str('.salt.loader.executors'),
'fun': salt.utils.stringutils.to_str('.salt.var.fun'),
'matcher': salt.utils.stringutils.to_str('.salt.matcher'),
'executors': salt.utils.stringutils.to_str('.salt.track.executors'),
'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'), }
def _prepare(self):
'''
Map opts for convenience
'''
self.opts = self.opts_store.value
self.proc_dir = salt.minion.get_proc_dir(self.opts['cachedir'])
self.serial = salt.payload.Serial(self.opts)
self.executors.value = {}
def _setup_jobber_stack(self):
'''
Setup and return the LaneStack and Yard used by the jobber yard
to communicate with the minion manor yard
'''
role = self.opts.get('id', '')
if not role:
emsg = ("Missing role required to setup Jobber Lane.")
log.error(emsg + "\n")
raise ValueError(emsg)
kind = self.opts['__role']
if kind not in kinds.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for Jobber lane.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind == 'minion':
lanename = "{0}_{1}".format(role, kind)
else:
emsg = ("Unsupported application kind = '{0}' for Jobber Lane.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
sockdirpath = self.opts['sock_dir']
name = 'jobber' + nacling.uuid(size=18)
stack = LaneStack(
name=name,
lanename=lanename,
sockdirpath=sockdirpath)
stack.Pk = raeting.PackKind.pack.value
# add remote for the manor yard
stack.addRemote(RemoteYard(stack=stack,
name='manor',
lanename=lanename,
dirpath=sockdirpath))
console.concise("Created Jobber Stack {0}\n".format(stack.name))
return stack
def _return_pub(self, msg, ret, stack):
'''
Send the return data back via the uxd socket
'''
route = {'src': (self.road_stack.value.local.name, stack.local.name, 'jid_ret'),
'dst': (msg['route']['src'][0], None, 'remote_cmd')}
mid = self.opts['id']
ret['cmd'] = '_return'
ret['id'] = mid
try:
oput = self.modules.value[ret['fun']].__outputter__
except (KeyError, AttributeError, TypeError):
pass
else:
if isinstance(oput, six.string_types):
ret['out'] = oput
msg = {'route': route, 'load': ret}
stack.transmit(msg, stack.fetchUidByName('manor'))
stack.serviceAll()
def action(self):
'''
Pull the queue for functions to execute
'''
while self.fun.value:
msg = self.fun.value.popleft()
data = msg.get('pub')
match = getattr(
self.matcher.value,
'{0}_match'.format(
data.get('tgt_type', 'glob')
)
)(data['tgt'])
if not match:
continue
if 'user' in data:
log.info(
'User %s Executing command %s with jid %s',
data['user'], data['fun'], data['jid']
)
else:
log.info(
'Executing command %s with jid %s',
data['fun'], data['jid']
)
log.debug('Command details %s', data)
if is_windows():
# SaltRaetNixJobber is not picklable. Pickling is necessary
# when spawning a process in Windows. Since the process will
# be spawned and joined on non-Windows platforms, instead of
# this, just run the function directly and absorb any thrown
# exceptions.
try:
self.proc_run(msg)
except Exception as exc:
log.error('Exception caught by jobber: %s', exc, exc_info=True)
else:
process = multiprocessing.Process(
target=self.proc_run,
kwargs={'msg': msg}
)
process.start()
process.join()
def proc_run(self, msg):
'''
Execute the run in a dedicated process
'''
data = msg['pub']
fn_ = os.path.join(self.proc_dir, data['jid'])
self.opts['__ex_id'] = data['jid']
salt.utils.process.daemonize_if(self.opts)
salt.transport.jobber_stack = stack = self._setup_jobber_stack()
# set up return destination from source
src_estate, src_yard, src_share = msg['route']['src']
salt.transport.jobber_estate_name = src_estate
salt.transport.jobber_yard_name = src_yard
sdata = {'pid': os.getpid()}
sdata.update(data)
with salt.utils.files.fopen(fn_, 'w+b') as fp_:
fp_.write(self.serial.dumps(sdata))
ret = {'success': False}
function_name = data['fun']
if function_name in self.modules.value:
try:
func = self.modules.value[data['fun']]
args, kwargs = salt.minion.load_args_and_kwargs(
func,
salt.utils.args.parse_input(
data['arg'],
no_parse=data.get('no_parse', [])),
data)
sys.modules[func.__module__].__context__['retcode'] = 0
executors = data.get('module_executors') or self.opts.get('module_executors', ['direct_call'])
if isinstance(executors, six.string_types):
executors = [executors]
elif not isinstance(executors, list) or not executors:
raise SaltInvocationError(
'Wrong executors specification: {0}. String or '
'non-empty list expected'.format(executors)
)
if self.opts.get('sudo_user', '') and executors[-1] != 'sudo':
executors[-1] = 'sudo' # replace
log.trace("Executors list %s", executors)
for name in executors:
fname = '{0}.execute'.format(name)
if fname not in self.module_executors.value:
raise SaltInvocationError("Executor '{0}' is not available".format(name))
return_data = self.module_executors.value[fname](self.opts, data, func, args, kwargs)
if return_data is not None:
break
if isinstance(return_data, types.GeneratorType):
ind = 0
iret = {}
for single in return_data:
if isinstance(single, dict) and isinstance(iret, list):
iret.update(single)
else:
if not iret:
iret = []
iret.append(single)
tag = tagify(
[data['jid'], 'prog', self.opts['id'], six.text_type(ind)],
'job')
event_data = {'return': single}
self._fire_master(event_data, tag) # Need to look into this
ind += 1
ret['return'] = iret
else:
ret['return'] = return_data
ret['retcode'] = sys.modules[func.__module__].__context__.get(
'retcode',
0
)
ret['success'] = True
except CommandNotFoundError as exc:
msg = 'Command required for \'{0}\' not found'.format(
function_name
)
log.debug(msg, exc_info=True)
ret['return'] = '{0}: {1}'.format(msg, exc)
except CommandExecutionError as exc:
log.error(
'A command in \'%s\' had a problem: %s',
function_name, exc,
exc_info_on_loglevel=logging.DEBUG
)
ret['return'] = 'ERROR: {0}'.format(exc)
except SaltInvocationError as exc:
log.error(
'Problem executing \'%s\': %s',
function_name, exc,
exc_info_on_loglevel=logging.DEBUG
)
ret['return'] = 'ERROR executing \'{0}\': {1}'.format(
function_name, exc
)
except TypeError as exc:
msg = ('TypeError encountered executing {0}: {1}. See '
'debug log for more info.').format(function_name, exc)
log.warning(msg, exc_info_on_loglevel=logging.DEBUG)
ret['return'] = msg
except Exception:
msg = 'The minion function caused an exception'
log.warning(msg, exc_info_on_loglevel=logging.DEBUG)
ret['return'] = '{0}: {1}'.format(msg, traceback.format_exc())
else:
ret['return'] = '\'{0}\' is not available.'.format(function_name)
ret['jid'] = data['jid']
ret['fun'] = data['fun']
ret['fun_args'] = data['arg']
self._return_pub(msg, ret, stack)
if data['ret']:
ret['id'] = self.opts['id']
for returner in set(data['ret'].split(',')):
try:
self.returners.value['{0}.returner'.format(
returner
)](ret)
except Exception as exc:
log.error('The return failed for job %s %s', data['jid'], exc)
console.concise("Closing Jobber Stack {0}\n".format(stack.name))
stack.server.close()
salt.transport.jobber_stack = None

28
salt/daemons/flo/maint.flo
View file

@ -1,28 +0,0 @@
# Maintenance process floscript
house master
framer jobcleaner be active first setup
frame setup
print Setup Maint
enter
do salt raet maint setup
go modules
frame modules
enter
do salt load modules
go fsclean
frame fsclean
enter
do salt raet maint fileserver clean
go start
frame start
do salt raet maint old jobs clear
framer backends be active first start
frame start
do salt raet maint backends update
framer scheduler be active first start
frame start
do salt schedule

152
salt/daemons/flo/maint.py
View file

@ -1,152 +0,0 @@
# -*- coding: utf-8 -*-
'''
Define the behaviors used in the maintenance process
'''
# pylint: disable=3rd-party-module-not-gated
from __future__ import absolute_import, print_function, unicode_literals
# Import python libs
import multiprocessing
import os
# Import ioflo libs
import ioflo.base.deeding
# Import salt libs
import salt.daemons.masterapi
import salt.fileserver
import salt.loader
import salt.utils.minions
import salt.utils.stringutils
@ioflo.base.deeding.deedify(
salt.utils.stringutils.to_str('SaltRaetMaintFork'),
ioinits={'opts': salt.utils.stringutils.to_str('.salt.opts'),
'proc_mgr': salt.utils.stringutils.to_str('.salt.usr.proc_mgr')})
def maint_fork(self):
'''
For off the maintinence process from the master router process
FloScript:
do salt raet maint fork at enter
'''
self.proc_mgr.value.add_process(Maintenance, args=(self.opts.value,))
class Maintenance(multiprocessing.Process):
'''
Start the maintinance process within ioflo
'''
def __init__(self, opts):
super(Maintenance, self).__init__()
self.opts = opts
def run(self):
'''
Spin up a worker, do this in s multiprocess
'''
behaviors = ['salt.daemons.flo']
preloads = [(salt.utils.stringutils.to_str('.salt.opts'), dict(value=self.opts))]
console_logdir = self.opts.get('ioflo_console_logdir', '')
if console_logdir:
consolepath = os.path.join(console_logdir, 'maintenance.log')
else: # empty means log to std out
consolepath = ''
ioflo.app.run.start(
name='maintenance',
period=float(self.opts['loop_interval']),
stamp=0.0,
real=self.opts['ioflo_realtime'],
filepath=self.opts['maintenance_floscript'],
behaviors=behaviors,
username="",
password="",
mode=None,
houses=None,
metas=None,
preloads=preloads,
verbose=int(self.opts['ioflo_verbose']),
consolepath=consolepath,
)
class SaltRaetMaintSetup(ioflo.base.deeding.Deed):
'''
Init loader objects used
FloScript:
do salt raet maint setup at enter
'''
Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'),
'fileserver': salt.utils.stringutils.to_str('.salt.loader.fileserver'),
'runners': salt.utils.stringutils.to_str('.salt.loader.runners'),
'pillargitfs': salt.utils.stringutils.to_str('.salt.loader.pillargitfs'),
'ckminions': salt.utils.stringutils.to_str('.salt.loader.ckminions')}
def action(self):
'''
Set up the objects used in the maint process
'''
self.fileserver.value = salt.fileserver.Fileserver(self.opts.value)
self.runners.value = salt.loader.runner(self.opts.value)
self.ckminions.value = salt.utils.minions.CkMinions(self.opts.value)
self.pillargitfs.value = salt.daemons.masterapi.init_git_pillar(
self.opts.value)
class SaltRaetMaintFileserverClean(ioflo.base.deeding.Deed):
'''
Clear the fileserver backend caches
FloScript:
do salt raet maint fileserver clean at enter
'''
Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts')}
def action(self):
'''
Clean!
'''
salt.daemons.masterapi.clean_fsbackend(self.opts.value)
class SaltRaetMaintOldJobsClear(ioflo.base.deeding.Deed):
'''
Iterate over the jobs directory and clean out the old jobs
FloScript:
do salt raet maint old jobs clear
'''
Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts')}
def action(self):
'''
Clear out the old jobs cache
'''
salt.daemons.masterapi.clean_old_jobs(self.opts.value)
class SaltRaetMaintBackendsUpdate(ioflo.base.deeding.Deed):
'''
Update the fileserver and external pillar caches
FloScript:
do salt raet maint backends update
'''
Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'),
'fileserver': salt.utils.stringutils.to_str('.salt.loader.fileserver'),
'pillargitfs': salt.utils.stringutils.to_str('.salt.loader.pillargitfs')}
def action(self):
'''
Update!
'''
for pillargit in self.pillargitfs.value:
pillargit.update()
salt.daemons.masterapi.fileserver_update(self.fileserver.value)

202
salt/daemons/flo/master.flo
View file

@ -1,202 +0,0 @@
# Salt Master Floscript
house master
framer masterudpstack be active first setup
# Begin the pre-flight checks
frame setup
enter
do salt raet cleanup
do salt raet road stack setup per inode ".salt.road.manor"
do salt raet manor lane setup per inode ".salt.lane.manor"
do salt raet process manager setup
go zmqstart
go spawnmaint
frame zmqstart
# Start the zmq ret port if configured
let me if .salt.etc.zmq_behavior
do salt zmq setup at enter
bid start zmqret
go spawnmaint
# Create the maintanence frame
frame spawnmaint
enter
do salt raet maint fork
go spawnworkers
# Start forking master workers
frame spawnworkers
enter
do salt raet worker fork
go spawnreactor
go spawneventreturn
go startengines
go start
frame spawnreactor
let me if .salt.etc.reactor
enter
do salt raet reactor fork
go spawneventreturn
go startengines
go start
frame spawneventreturn
let me if .salt.etc.event_return
enter
do salt raet event return fork
go startengines
go start
frame startengines
let me if .salt.etc.engines
enter
do salt raet setup engines
go start
frame start
# Start the message receive framer
bid start inbound
# Start the cluster bootstrap framer
bid start bootstrap
# Start the message receive framer
bid start uxdrouter
# Start the event framer
bid start events
# Start the presence framer
bid start presence
# Start the publish framer
bid start publish
# Start the manage framer
bid start manager
# Start the outbound framer
bid start outbound
exit
do salt raet road stack closer per inode ".salt.road.manor."
do salt raet lane stack closer per inode ".salt.lane.manor."
#########################################
# Main RAET Behaviors #
#########################################
# Inbound framer
framer inbound be inactive first start
frame start
do salt raet road stack service rx
do salt raet lane stack service rx
# Bootstrap framer
framer bootstrap be inactive first setup
frame setup
enter
do salt raet road clustered per inode ".salt.road.manor."
go clustermaster
go quit
frame clustermaster
let if salt.road.manor.cluster.clustered
print Setting Up Master Cluster ....
do salt raet road usher master setup per inode ".salt.road.manor."
go join
frame join
print Joining...
enter
do salt raet road stack joiner per inode ".salt.road.manor."
recur
do salt raet road stack joined per inode ".salt.road.manor."
do salt raet road stack rejected per inode ".salt.road.manor."
go next if joined in .salt.road.manor.status
#go abort if rejected in .salt.road.manor.status
frame joined
print Joined
go next if elapsed >= 0.5
frame allow
print Allowing...
enter
do salt raet road stack allower per inode ".salt.road.manor."
recur
do salt raet road stack allowed per inode ".salt.road.manor."
go next if allowed in .salt.road.manor.status
frame allowed
print Allowed
go next
frame clustering
print Cluster Setup ...
do salt raet road cluster load setup
go next
frame quit
bid stop me
frame abort
bid stop all
# Router framer
framer uxdrouter be inactive first start
frame start
do salt raet router master
# Event bus framer
framer events be inactive first start
frame start
do salt raet eventer master
do salt raet stats eventer master
# Presence framer
framer presence be inactive first start
frame start
go multiheaded
go raet
frame raet
do salt raet presenter
frame multiheaded
let me if .salt.etc.zmq_behavior
do salt zmq crypticle setup at enter
do salt zmq publisher
do salt raet presenter
# Publisher framer
framer publish be inactive first setup
frame setup
go dummy_publisher
go raet_publisher
frame dummy_publisher
let me if .salt.etc.dummy_pub
do salt dummy publisher
frame raet_publisher
do salt raet publisher
# Manager framer
framer manager be inactive first start at 10.0
frame start
do salt raet road stack manager per inode ".salt.road.manor"
# Outbound framer
framer outbound be inactive first start
frame start
do salt raet lane stack service tx
do salt raet road stack service tx
#########################################
# Main ZMQ Behaviors #
#########################################
framer zmqret be inactive first start
frame start
enter
do salt zmq ret fork

187
salt/daemons/flo/minion.flo
View file

@ -1,187 +0,0 @@
# Salt Minion floscript
house minion
framer minionudpstack be active first setup
# Begin the pre-flight checks
frame setup
enter
do salt raet cleanup
do salt raet road stack setup per inode ".salt.road.manor"
do salt raet manor lane setup
do salt raet process manager setup
go loadmodules
# Load the minion mods
frame loadmodules
do salt load modules at enter
go setupmatcher
frame setupmatcher
do salt raet setup matcher at enter
go setupbeacon
frame setupbeacon
do salt raet setup beacon at enter
go spawnreactor
go startengines
go start
frame spawnreactor
let me if .salt.etc.reactor
enter
do salt raet reactor fork
go startengines
go start
frame startengines
let me if .salt.etc.engines
enter
do salt raet setup engines
go start
# OK, let's start the minion up
frame start
# Start the inbound framer
bid start inbound
# Start the bootstrap framer
bid start bootstrap
# Start the eventing framer
bid start eventing
# Start the functionmanage framer
bid start functionmanager
# Start the outbound framer
bid start outbound
# Start the scheduler
bid start scheduler
# Cleanup on exit
exit
do salt raet road stack closer per inode ".salt.road.manor."
do salt raet lane stack closer per inode ".salt.lane.manor."
# Framer for handling inbound traffic
framer inbound be inactive first start
frame start
do salt raet road stack service rx
do salt raet lane stack service rx
framer bootstrap be inactive first setup
frame setup
enter
do salt raet road clustered per inode ".salt.road.manor."
do salt raet road usher minion setup per inode ".salt.road.manor."
go clustermaster
go multimaster
frame clustermaster
let if salt.road.manor.cluster.clustered
print Setting Up Master Cluster ....
go join
frame multimaster
print Setting Up Master or MultiMaster
go join
frame join
print Joining...
enter
do salt raet road stack joiner per inode ".salt.road.manor."
recur
do salt raet road stack joined per inode ".salt.road.manor."
do salt raet road stack rejected per inode ".salt.road.manor."
go next if joined in .salt.road.manor.status
go abort if rejected in .salt.road.manor.status
frame joined
print Joined
go next if elapsed >= 0.5
frame allow
print Allowing...
enter
do salt raet road stack allower per inode ".salt.road.manor."
recur
do salt raet road stack allowed per inode ".salt.road.manor."
go next if allowed in .salt.road.manor.status
frame allowed
print Allowed
go next if elapsed >= 0.5
frame clustering
print Cluster Setup ...
do salt raet road cluster load setup
go next
frame pillar
print Pillaring
enter
do salt load pillar
go loading
frame loading
print Loading
enter
do salt load modules
go latestart
frame latestart
# Start late frames that need the pillar/modules to be available
# Start the master events loop
bid start masterevents
# Start Beacon
bid start beacon
go router
frame router
# start the manager framer
bid start manager #start alive presence from minion side
do salt raet router minion
go pillar if .salt.var.pillar_refresh
go loading if .salt.var.module_refresh
frame abort
bid stop all
framer eventing be inactive first event
frame event
do salt raet eventer
do salt raet stats eventer minion
framer functionmanager be inactive first setup
frame setup
go shell
go nix
frame shell
let me if .salt.etc.shell_jobber
do salt raet shell jobber
do salt raet shell jobber check
frame nix
do salt raet nix jobber
framer manager be inactive first start at 10.0
frame start
do salt raet road stack manager per inode ".salt.road.manor"
framer beacon be inactive first start
frame start
do salt raet beacon
framer masterevents be inactive first start
frame start
do salt raet master events
# Framer for handling outbound traffic
framer outbound be inactive first start
frame start
do salt raet lane stack service tx
do salt raet road stack service tx
framer scheduler be inactive first start
frame start
do salt schedule

40
salt/daemons/flo/reactor.py
View file

@ -1,40 +0,0 @@
# -*- coding: utf-8 -*-
'''
Start the reactor!
'''
# pylint: disable=3rd-party-module-not-gated
from __future__ import absolute_import, print_function, unicode_literals
# Import salt libs
import salt.utils.reactor
import salt.utils.event
import salt.utils.stringutils
# Import ioflo libs
import ioflo.base.deeding
@ioflo.base.deeding.deedify(
salt.utils.stringutils.to_str('SaltRaetReactorFork'),
ioinits={
'opts': salt.utils.stringutils.to_str('.salt.opts'),
'proc_mgr': salt.utils.stringutils.to_str('.salt.usr.proc_mgr')})
def reactor_fork(self):
'''
Add a reactor object to the process manager
'''
self.proc_mgr.value.add_process(
salt.utils.reactor.Reactor,
args=(self.opts.value,))
@ioflo.base.deeding.deedify(
salt.utils.stringutils.to_str('SaltRaetEventReturnFork'),
ioinits={
'opts': salt.utils.stringutils.to_str('.salt.opts'),
'proc_mgr': salt.utils.stringutils.to_str('.salt.usr.proc_mgr')})
def event_return_fork(self):
'''
Add a reactor object to the process manager
'''
self.proc_mgr.value.add_process(
salt.utils.event.EventReturn,
args=(self.opts.value,))

30
salt/daemons/flo/worker.flo
View file

@ -1,30 +0,0 @@
# Salt Master Worker Floscript.
#
# This controls a single worker proc, many are started based on the value in worker_threads
house worker
framer uxdrouter be active first setup
frame setup
do salt raet worker setup at enter
go zmqsetup
go raetstart
frame zmqsetup
# Only enter if configured to run dual head
let me if .salt.etc.zmq_behavior
bid start zmqworker
go raetstart
frame raetstart
bid start raetworker
framer raetworker be inactive first start
frame start
do salt raet worker router
exit
do salt raet lane stack closer per inode ".salt.lane.manor."
framer zmqworker be inactive first start
frame start
do salt zmq worker

249
salt/daemons/flo/worker.py
View file

@ -1,249 +0,0 @@
# -*- coding: utf-8 -*-
'''
The core behaviors used by minion and master
'''
# pylint: disable=W0232
# pylint: disable=3rd-party-module-not-gated
from __future__ import absolute_import, print_function, unicode_literals
# Import python libs
import time
import os
import multiprocessing
import logging
from salt.ext.six.moves import range
# Import salt libs
import salt.daemons.flo
import salt.daemons.masterapi
from raet import raeting
from raet.lane.stacking import LaneStack
from raet.lane.yarding import RemoteYard
import salt.utils.kinds as kinds
import salt.utils.stringutils
# Import ioflo libs
import ioflo.base.deeding
log = logging.getLogger(__name__)
# convert to set once list is larger than about 3 because set hashes
INHIBIT_RETURN = [] # ['_return'] # cmd for which we should not send return
@ioflo.base.deeding.deedify(
salt.utils.stringutils.to_str('SaltRaetWorkerFork'),
ioinits={
'opts': salt.utils.stringutils.to_str('.salt.opts'),
'proc_mgr': salt.utils.stringutils.to_str('.salt.usr.proc_mgr'),
'worker_verify': salt.utils.stringutils.to_str('.salt.var.worker_verify'),
'access_keys': salt.utils.stringutils.to_str('.salt.access_keys'),
'mkey': salt.utils.stringutils.to_str('.salt.var.zmq.master_key'),
'aes': salt.utils.stringutils.to_str('.salt.var.zmq.aes')})
def worker_fork(self):
'''
Fork off the worker procs
FloScript:
do salt raet worker fork at enter
'''
for index in range(int(self.opts.value['worker_threads'])):
time.sleep(0.01)
self.proc_mgr.value.add_process(
Worker,
args=(
self.opts.value,
index + 1,
self.worker_verify.value,
self.access_keys.value,
self.mkey.value,
self.aes.value
)
)
class Worker(multiprocessing.Process):
'''
Create an ioflo worker in a separate process
'''
def __init__(self, opts, windex, worker_verify, access_keys, mkey, aes):
super(Worker, self).__init__()
self.opts = opts
self.windex = windex
self.worker_verify = worker_verify
self.access_keys = access_keys
self.mkey = mkey
self.aes = aes
def run(self):
'''
Spin up a worker, do this in multiprocess
windex is worker index
'''
self.opts['__worker'] = True
behaviors = ['salt.daemons.flo']
preloads = [(salt.utils.stringutils.to_str('.salt.opts'), dict(value=self.opts)),
(salt.utils.stringutils.to_str('.salt.var.worker_verify'),
dict(value=self.worker_verify))]
preloads.append((salt.utils.stringutils.to_str('.salt.var.fork.worker.windex'),
dict(value=self.windex)))
preloads.append((salt.utils.stringutils.to_str('.salt.var.zmq.master_key'),
dict(value=self.mkey)))
preloads.append((salt.utils.stringutils.to_str('.salt.var.zmq.aes'), dict(value=self.aes)))
preloads.append((salt.utils.stringutils.to_str('.salt.access_keys'),
dict(value=self.access_keys)))
preloads.extend(salt.daemons.flo.explode_opts(self.opts))
console_logdir = self.opts.get('ioflo_console_logdir', '')
if console_logdir:
consolepath = os.path.join(console_logdir, "worker_{0}.log".format(self.windex))
else: # empty means log to std out
consolepath = ''
ioflo.app.run.start(
name='worker{0}'.format(self.windex),
period=float(self.opts['ioflo_period']),
stamp=0.0,
real=self.opts['ioflo_realtime'],
filepath=self.opts['worker_floscript'],
behaviors=behaviors,
username='',
password='',
mode=None,
houses=None,
metas=None,
preloads=preloads,
verbose=int(self.opts['ioflo_verbose']),
consolepath=consolepath,
)
class SaltRaetWorkerSetup(ioflo.base.deeding.Deed):
'''
FloScript:
do salt raet worker setup at enter
'''
Ioinits = {
'opts': salt.utils.stringutils.to_str('.salt.opts'),
'windex': salt.utils.stringutils.to_str('.salt.var.fork.worker.windex'),
'access_keys': salt.utils.stringutils.to_str('.salt.access_keys'),
'remote_loader': salt.utils.stringutils.to_str('.salt.loader.remote'),
'local_loader': salt.utils.stringutils.to_str('.salt.loader.local'),
'inode': salt.utils.stringutils.to_str('.salt.lane.manor.'),
'stack': salt.utils.stringutils.to_str('stack'),
'local': {'ipath': salt.utils.stringutils.to_str('local'),
'ival': {'lanename': 'master'}}
}
def action(self):
'''
Set up the uxd stack and behaviors
'''
name = "worker{0}".format(self.windex.value)
# master application kind
kind = self.opts.value['__role']
if kind not in kinds.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for Master Worker.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.master],
kinds.APPL_KIND_NAMES[kinds.applKinds.syndic]]:
lanename = 'master'
else: # workers currently are only supported for masters
emsg = ("Invalid application kind '{0}' for Master Worker.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
sockdirpath = self.opts.value['sock_dir']
self.stack.value = LaneStack(
name=name,
lanename=lanename,
sockdirpath=sockdirpath)
self.stack.value.Pk = raeting.PackKind.pack.value
manor_yard = RemoteYard(
stack=self.stack.value,
name='manor',
lanename=lanename,
dirpath=sockdirpath)
self.stack.value.addRemote(manor_yard)
self.remote_loader.value = salt.daemons.masterapi.RemoteFuncs(
self.opts.value)
self.local_loader.value = salt.daemons.masterapi.LocalFuncs(
self.opts.value,
self.access_keys.value)
init = {}
init['route'] = {
'src': (None, self.stack.value.local.name, None),
'dst': (None, manor_yard.name, 'worker_req')
}
self.stack.value.transmit(init, self.stack.value.fetchUidByName(manor_yard.name))
self.stack.value.serviceAll()
def __del__(self):
self.stack.server.close()
class SaltRaetWorkerRouter(ioflo.base.deeding.Deed):
'''
FloScript:
do salt raet worker router
'''
Ioinits = {
'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'),
'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'),
'opts': salt.utils.stringutils.to_str('.salt.opts'),
'worker_verify': salt.utils.stringutils.to_str('.salt.var.worker_verify'),
'remote_loader': salt.utils.stringutils.to_str('.salt.loader.remote'),
'local_loader': salt.utils.stringutils.to_str('.salt.loader.local'),
}
def action(self):
'''
Read in a command and execute it, send the return back up to the
main master process
'''
self.lane_stack.value.serviceAll()
while self.lane_stack.value.rxMsgs:
msg, sender = self.lane_stack.value.rxMsgs.popleft()
try:
s_estate, s_yard, s_share = msg['route']['src']
d_estate, d_yard, d_share = msg['route']['dst']
except (ValueError, IndexError):
log.error('Received invalid message: %s', msg)
return
log.debug("**** Worker Router rxMsg\nmsg=%s", msg)
if 'load' in msg:
cmd = msg['load'].get('cmd')
if not cmd:
continue
elif cmd.startswith('__'):
continue
ret = {}
if d_share == 'remote_cmd':
if hasattr(self.remote_loader.value, cmd):
ret['return'] = getattr(self.remote_loader.value, cmd)(msg['load'])
elif d_share == 'local_cmd':
if hasattr(self.local_loader.value, cmd):
ret['return'] = getattr(self.local_loader.value, cmd)(msg['load'])
else:
ret = {'error': 'Invalid request'}
if cmd == 'publish' and 'pub' in ret.get('return', {}):
r_share = 'pub_ret'
ret['__worker_verify'] = self.worker_verify.value
else:
r_share = s_share
if cmd not in INHIBIT_RETURN:
ret['route'] = {
'src': (None, self.lane_stack.value.local.name, None),
'dst': (s_estate, s_yard, r_share)
}
self.lane_stack.value.transmit(ret,
self.lane_stack.value.fetchUidByName('manor'))
self.lane_stack.value.serviceAll()

249
salt/daemons/flo/zero.py
View file

@ -1,249 +0,0 @@
# -*- coding: utf-8 -*-
'''
IoFlo behaviors for running a ZeroMQ based master
'''
# pylint: disable=W0232
# pylint: disable=3rd-party-module-not-gated
# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
import os
import logging
import hashlib
import multiprocessing
import errno
# Import ioflo libs
import ioflo.base.deeding
# Import third party libs
from salt.utils.zeromq import zmq
import salt.master
import salt.crypt
import salt.daemons.masterapi
import salt.payload
import salt.utils.stringutils
log = logging.getLogger(__name__)
class SaltZmqSetup(ioflo.base.deeding.Deed):
'''
do salt zmq setup at enter
Setup shares
.salt.var.zmq.master_key
.salt.var.zmq.aet share
This behavior must be run before any other zmq related
'''
Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'),
'mkey': salt.utils.stringutils.to_str('.salt.var.zmq.master_key'),
'aes': salt.utils.stringutils.to_str('.salt.var.zmq.aes')}
def action(self):
'''
Assign master key to .salt.var.zmq.master_key
Copy opts['aes'] to .salt.var.zmq.aes
'''
self.mkey.value = salt.crypt.MasterKeys(self.opts.value)
self.aes.value = self.opts.value['aes']
@ioflo.base.deeding.deedify(
salt.utils.stringutils.to_str('SaltZmqRetFork'),
ioinits={
'opts': salt.utils.stringutils.to_str('.salt.opts'),
'proc_mgr': salt.utils.stringutils.to_str('.salt.usr.proc_mgr'),
'mkey': salt.utils.stringutils.to_str('.salt.var.zmq.master_key'),
'aes': salt.utils.stringutils.to_str('.salt.var.zmq.aes')})
def zmq_ret_fork(self):
'''
Create the forked process for the ZeroMQ Ret Port
'''
self.proc_mgr.value.add_process(
ZmqRet,
args=(
self.opts.value,
self.mkey.value,
self.aes.value))
class ZmqRet(multiprocessing.Process):
'''
Create the forked process for the ZeroMQ Ret Port
'''
def __init__(self, opts, mkey, aes):
self.opts = opts
self.mkey = mkey
self.aes = aes
super(ZmqRet, self).__init__()
def run(self):
'''
Start the ret port binding
'''
self.context = zmq.Context(self.opts['worker_threads'])
self.uri = 'tcp://{interface}:{ret_port}'.format(**self.opts)
log.info('ZMQ Ret port binding to %s', self.uri)
self.clients = self.context.socket(zmq.ROUTER)
if self.opts['ipv6'] is True and hasattr(zmq, 'IPV4ONLY'):
# IPv6 sockets work for both IPv6 and IPv4 addresses
self.clients.setsockopt(zmq.IPV4ONLY, 0)
try:
self.clients.setsockopt(zmq.HWM, self.opts['rep_hwm'])
except AttributeError:
self.clients.setsockopt(zmq.SNDHWM, self.opts['rep_hwm'])
self.clients.setsockopt(zmq.RCVHWM, self.opts['rep_hwm'])
self.clients.setsockopt(zmq.BACKLOG, self.opts['zmq_backlog'])
self.workers = self.context.socket(zmq.DEALER)
self.w_uri = 'ipc://{0}'.format(
os.path.join(self.opts['sock_dir'], 'workers.ipc')
)
log.info('Setting up the master communication server')
self.clients.bind(self.uri)
self.workers.bind(self.w_uri)
while True:
try:
zmq.device(zmq.QUEUE, self.clients, self.workers)
except zmq.ZMQError as exc:
if exc.errno == errno.EINTR:
continue
raise exc
class SaltZmqCrypticleSetup(ioflo.base.deeding.Deed):
'''
Setup the crypticle for the salt zmq publisher behavior
do salt zmq crypticle setup at enter
'''
Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'),
'aes': salt.utils.stringutils.to_str('.salt.var.zmq.aes'),
'crypticle': salt.utils.stringutils.to_str('.salt.var.zmq.crypticle')}
def action(self):
'''
Initializes zmq
Put here so only runs initialization if we want multi-headed master
'''
self.crypticle.value = salt.crypt.Crypticle(
self.opts.value,
self.opts.value.get('aes'))
class SaltZmqPublisher(ioflo.base.deeding.Deed):
'''
The zeromq publisher
do salt zmq publisher
Must run the deed
do salt zmq publisher setup
before this deed
'''
Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'),
'publish': salt.utils.stringutils.to_str('.salt.var.publish'),
'zmq_behavior': salt.utils.stringutils.to_str('.salt.etc.zmq_behavior'),
'aes': salt.utils.stringutils.to_str('.salt.var.zmq.aes'),
'crypticle': salt.utils.stringutils.to_str('.salt.var.zmq.crypticle')}
def _prepare(self):
'''
Set up tracking value(s)
'''
if not zmq:
return
self.created = False
self.serial = salt.payload.Serial(self.opts.value)
def action(self):
'''
Create the publish port if it is not available and then publish the
messages on it
'''
if not self.zmq_behavior:
return
if not self.created:
self.context = zmq.Context(1)
self.pub_sock = self.context.socket(zmq.PUB)
# if 2.1 >= zmq < 3.0, we only have one HWM setting
try:
self.pub_sock.setsockopt(zmq.HWM, self.opts.value.get('pub_hwm', 1000))
# in zmq >= 3.0, there are separate send and receive HWM settings
except AttributeError:
self.pub_sock.setsockopt(zmq.SNDHWM, self.opts.value.get('pub_hwm', 1000))
self.pub_sock.setsockopt(zmq.RCVHWM, self.opts.value.get('pub_hwm', 1000))
if self.opts.value['ipv6'] is True and hasattr(zmq, 'IPV4ONLY'):
# IPv6 sockets work for both IPv6 and IPv4 addresses
self.pub_sock.setsockopt(zmq.IPV4ONLY, 0)
self.pub_sock.setsockopt(zmq.BACKLOG, self.opts.get('zmq_backlog', 1000))
self.pub_uri = 'tcp://{interface}:{publish_port}'.format(**self.opts.value)
log.info('Starting the Salt ZeroMQ Publisher on %s', self.pub_uri)
self.pub_sock.bind(self.pub_uri)
self.created = True
# Don't pop the publish messages! The raet behavior still needs them
try:
for package in self.publish.value:
payload = {'enc': 'aes'}
payload['load'] = self.crypticle.value.dumps(package['return']['pub'])
if self.opts.value['sign_pub_messages']:
master_pem_path = os.path.join(self.opts.value['pki_dir'], 'master.pem')
log.debug('Signing data packet for publish')
payload['sig'] = salt.crypt.sign_message(master_pem_path, payload['load'])
send_payload = self.serial.dumps(payload)
if self.opts.value['zmq_filtering']:
# if you have a specific topic list, use that
if package['return']['pub']['tgt_type'] == 'list':
for topic in package['return']['pub']['tgt']:
# zmq filters are substring match, hash the topic
# to avoid collisions
htopic = hashlib.sha1(topic).hexdigest()
self.pub_sock.send(htopic, flags=zmq.SNDMORE)
self.pub_sock.send(send_payload)
# otherwise its a broadcast
else:
self.pub_sock.send('broadcast', flags=zmq.SNDMORE)
self.pub_sock.send(send_payload)
else:
self.pub_sock.send(send_payload)
except zmq.ZMQError as exc:
if exc.errno == errno.EINTR:
return
raise exc
class SaltZmqWorker(ioflo.base.deeding.Deed):
'''
The zeromq behavior for the workers
'''
Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts'),
'key': salt.utils.stringutils.to_str('.salt.access_keys'),
'aes': salt.utils.stringutils.to_str('.salt.var.zmq.aes')}
def _prepare(self):
'''
Create the initial seting value for the worker
'''
self.created = False
def action(self):
'''
Create the master MWorker if it is not present, then iterate over the
connection with the ioflo sequence
'''
if not self.created:
crypticle = salt.crypt.Crypticle(self.opts.value, self.aes.value)
self.worker = salt.master.FloMWorker(
self.opts.value,
self.key.value,
)
self.worker.setup()
self.created = True
log.info('Started ZMQ worker')
self.worker.handle_request()

279
salt/daemons/salting.py
View file

@ -1,279 +0,0 @@
# -*- coding: utf-8 -*-
'''
salting.py module of salt specific interfaces to raet
'''
from __future__ import absolute_import, print_function, unicode_literals
# pylint: skip-file
# pylint: disable=W0611
# Import Python libs
import os
# Import ioflo libs
from ioflo.aid.odicting import odict
from ioflo.base.consoling import getConsole
console = getConsole()
from raet import raeting, nacling
from raet.keeping import Keep
from salt.key import RaetKey
import salt.utils.kinds as kinds
class SaltKeep(Keep):
'''
RAET protocol estate on road data persistence for a given estate
road specific data
road/
keep/
stackname/
local/
estate.ext
remote/
estate.name.ext
estate.name.ext
'''
LocalFields = ['name', 'uid', 'ha', 'iha', 'natted', 'fqdn', 'dyned', 'sid',
'puid', 'aha', 'role', 'sighex','prihex']
LocalDumpFields = ['name', 'uid', 'ha', 'iha', 'natted', 'fqdn', 'dyned', 'sid',
'puid', 'aha', 'role']
RemoteFields = ['name', 'uid', 'fuid', 'ha', 'iha', 'natted', 'fqdn', 'dyned',
'sid', 'main', 'kind', 'joined',
'role', 'acceptance', 'verhex', 'pubhex']
RemoteDumpFields = ['name', 'uid', 'fuid', 'ha', 'iha', 'natted', 'fqdn', 'dyned',
'sid', 'main', 'kind', 'joined', 'role']
Auto = raeting.AutoMode.never.value #auto accept
def __init__(self, opts, prefix='estate', basedirpath='', auto=None, **kwa):
'''
Setup RoadKeep instance
'''
basedirpath = basedirpath or os.path.join(opts['cache_dir'], 'raet')
super(SaltKeep, self).__init__(prefix=prefix, basedirpath=basedirpath, **kwa)
self.auto = (auto if auto is not None else
(raeting.AutoMode.always.value if opts['open_mode'] else
(raeting.AutoMode.once.value if opts['auto_accept'] else
raeting.AutoMode.never.value)))
self.saltRaetKey = RaetKey(opts)
def clearAllDir(self):
'''
Clear all keep directories
'''
super(SaltKeep, self).clearAllDir()
self.clearRoleDir()
def clearRoleDir(self):
'''
Clear the Role directory
'''
self.saltRaetKey.delete_pki_dir()
def loadLocalRoleData(self):
'''
Load and return the role data
'''
keydata = self.saltRaetKey.read_local()
if not keydata:
keydata = odict([('sign', None), ('priv', None)])
data = odict([('sighex', keydata['sign']),
('prihex', keydata['priv'])])
return data
def clearLocalRoleData(self):
'''
Clear the local file
'''
self.saltRaetKey.delete_local()
def clearLocalRoleDir(self):
'''
Clear the Local Role directory
'''
self.saltRaetKey.delete_pki_dir()
def loadLocalData(self):
'''
Load and Return the data from the local estate
'''
data = super(SaltKeep, self).loadLocalData()
if not data:
return None
roleData = self.loadLocalRoleData() # if not present defaults None values
data.update([('sighex', roleData.get('sighex')),
('prihex', roleData.get('prihex'))])
return data
def loadRemoteData(self, name):
'''
Load and Return the data from the remote file
'''
data = super(SaltKeep, self).loadRemoteData(name)
if not data:
return None
mid = data['role']
for status in [acceptance.name for acceptance in Acceptance]:
keydata = self.saltRaetKey.read_remote(mid, status)
if keydata:
break
if not keydata:
data.update([('acceptance', None),
('verhex', None),
('pubhex', None)])
else:
data.update(acceptance=raeting.Acceptance[status].value,
verhex=keydata['verify'],
pubhex=keydata['pub'])
return data
def loadAllRemoteData(self):
'''
Load and Return the data from the all the remote estate files
'''
keeps = super(SaltKeep, self).loadAllRemoteData()
for name, data in keeps.items():
keeps[name].update([('acceptance', None),
('verhex', None),
('pubhex', None)])
for status, mids in self.saltRaetKey.list_keys().items():
for mid in mids:
keydata = self.saltRaetKey.read_remote(mid, status)
if keydata:
for name, data in keeps.items():
if data['role'] == mid:
keeps[name].update(
[('acceptance', raeting.Acceptance[status].value),
('verhex', keydata['verify']),
('pubhex', keydata['pub'])])
return keeps
def clearRemoteRoleData(self, role):
'''
Clear data from the role data file
'''
self.saltRaetKey.delete_key(role) #now delete role key file
def clearAllRemoteRoleData(self):
'''
Remove all the role data files
'''
self.saltRaetKey.delete_all()
def clearRemoteRoleDir(self):
'''
Clear the Remote Role directory
'''
self.saltRaetKey.delete_pki_dir()
def dumpLocal(self, local):
'''
Dump local estate
'''
data = odict([
('name', local.name),
('uid', local.uid),
('ha', local.ha),
('iha', local.iha),
('natted', local.natted),
('fqdn', local.fqdn),
('dyned', local.dyned),
('sid', local.sid),
('puid', local.stack.puid),
('aha', local.stack.aha),
('role', local.role),
])
if self.verifyLocalData(data, localFields =self.LocalDumpFields):
self.dumpLocalData(data)
self.saltRaetKey.write_local(local.priver.keyhex, local.signer.keyhex)
def dumpRemote(self, remote):
'''
Dump remote estate
'''
data = odict([
('name', remote.name),
('uid', remote.uid),
('fuid', remote.fuid),
('ha', remote.ha),
('iha', remote.iha),
('natted', remote.natted),
('fqdn', remote.fqdn),
('dyned', remote.dyned),
('sid', remote.sid),
('main', remote.main),
('kind', remote.kind),
('joined', remote.joined),
('role', remote.role),
])
if self.verifyRemoteData(data, remoteFields=self.RemoteDumpFields):
self.dumpRemoteData(data, remote.name)
if remote.pubber.keyhex and remote.verfer.keyhex:
# kludge to persist the keys since no way to write
self.saltRaetKey.status(remote.role,
remote.pubber.keyhex,
remote.verfer.keyhex)
def statusRemote(self, remote, dump=True):
'''
Calls .statusRole on remote role and keys and updates remote.acceptance
dump indicates if statusRole should update persisted values when
appropriate.
Returns status
Where status is acceptance status of role and keys
and has value from raeting.acceptances
'''
status = self.statusRole(role=remote.role,
verhex=remote.verfer.keyhex,
pubhex=remote.pubber.keyhex,
dump=dump)
remote.acceptance = status
return status
def statusRole(self, role, verhex, pubhex, dump=True):
'''
Returns status
Where status is acceptance status of role and keys
and has value from raeting.acceptances
'''
status = raeting.Acceptance[self.saltRaetKey.status(role,
pubhex,
verhex)].value
return status
def rejectRemote(self, remote):
'''
Set acceptance status to rejected
'''
mid = remote.role
self.saltRaetKey.reject(match=mid, include_accepted=True)
remote.acceptance = raeting.Acceptance.rejected.value
def pendRemote(self, remote):
'''
Set acceptance status to pending
'''
pass
def acceptRemote(self, remote):
'''
Set acceptance status to accepted
'''
mid = remote.role
self.saltRaetKey.accept(match=mid, include_rejected=True)
remote.acceptance = raeting.Acceptance.accepted.value

46
salt/daemons/test/__init__.py
View file

@ -1,46 +0,0 @@
# -*- coding: utf-8 -*-
'''
salt daemons raet unit test package
To run the unittests:
from salt.daemons import test
test.run()
'''
from __future__ import absolute_import, print_function, unicode_literals
import sys
# pylint: disable=blacklisted-import
if sys.version_info < (2, 7):
import unittest2 as unittest
else:
import unittest
# pylint: enable=blacklisted-import
import os
from ioflo.base.consoling import getConsole
console = getConsole()
console.reinit(verbosity=console.Wordage.concise)
def run(start=None):
'''
Run unittests starting at directory given by start
Default start is the location of the raet package
'''
top = os.path.dirname(os.path.dirname(os.path.abspath(
sys.modules.get(__name__).__file__)))
if not start:
start = top
console.terse("\nRunning all salt.daemons unit tests in '{0}', starting at '{1}'\n".format(top, start))
loader = unittest.TestLoader()
suite = loader.discover(start, 'test_*.py', top)
unittest.TextTestRunner(verbosity=2).run(suite)
if __name__ == "__main__":
run()

26
salt/daemons/test/master.flo
View file

@ -1,26 +0,0 @@
# Raet Test FloScript
house master
init .salt.road.manor.local to role "master" host "" port 7530 main true
framer masterudpstack be active first start
frame start
do salt raet cleanup at enter
do salt raet road stack setup per inode ".salt.road.manor" at enter
bid start service
do raet road stack closer per inode ".salt.road.manor." at exit
framer printer be active first start
frame start
do salt raet road stack printer per inode ".salt.road.manor."
timeout 20
frame abort
bid stop all
framer service be inactive first start
frame start
do salt raet road stack service

66
salt/daemons/test/minion.flo
View file

@ -1,66 +0,0 @@
# Raet Test FloScript
house minion
init .salt.road.manor.local to name "minion" host "" port 7531 main false
framer minionudpstack be active first start
frame start
enter
do salt raet road stack setup per inode ".salt.road.manor"
exit
do salt raet road stack closer per inode ".salt.road.manor."
framer bootstrap be active first setup
frame setup
enter
do salt raet road usher minion setup per inode ".salt.road.manor."
go join
frame join
print Joining...
enter
do salt raet road stack joiner per inode ".salt.road.manor."
recur
do raet road stack joined per inode ".salt.road.manor."
go next if joined in .salt.road.manor.status
go abort if elapsed >= 10
frame joined
print Joined
go next
frame allow
print Allowing...
enter
do salt raet road stack allower per inode ".salt.road.manor."
recur
do salt raet road stack allowed per inode ".salt.road.manor."
go next if allowed in .salt.road.manor.status
go abort if elapsed >= 5
frame allowed
print Allowed
go next
frame message
print Messaging...
enter
do raet road stack messenger with contents "Minion 1 Hello" code 15 \
per inode ".salt.road.manor."
go next
frame idle
print Idling...
do raet road stack idled per inode ".salt.road.manor."
go abort if idled in .salt.road.manor.status
frame abort
bid stop all
framer service be active first start
frame start
do salt raet road stack service

6
salt/daemons/test/plan/__init__.py
View file

@ -1,6 +0,0 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
from . import actors
__all__ = ['actors']

829
salt/daemons/test/plan/actors.py
View file

@ -1,829 +0,0 @@
# -*- coding: utf-8 -*-
'''
Test behaviors used by test plans
'''
from __future__ import absolute_import, print_function, unicode_literals
import os
import stat
import time
from collections import deque
# Import ioflo libs
import ioflo.base.deeding
from ioflo.aid.odicting import odict
from ioflo.base.consoling import getConsole
console = getConsole()
from raet import raeting, nacling
from raet.lane.stacking import LaneStack
from raet.lane.yarding import RemoteYard
from raet.road.estating import RemoteEstate
from raet.road.stacking import RoadStack
from raet.stacking import Stack
import salt.utils.stringutils
from salt.daemons import salting
from salt.utils.event import tagify
class DeedTestWrapper(object):
def assertTrue(self, condition):
if not condition:
self.failure.value = 'Fail'
raise Exception("Test Failed")
def createStack(ip):
stack = Stack()
stack.ha = (ip, '1234')
return stack
class TestOptsSetup(ioflo.base.deeding.Deed):
'''
Setup opts share
'''
Ioinits = {'opts': salt.utils.stringutils.to_str('.salt.opts')}
def action(self):
'''
Register presence requests
Iterate over the registered presence yards and fire!
'''
pkiDirpath = os.path.join('/tmp', 'raet', 'test', self.role, 'pki')
if not os.path.exists(pkiDirpath):
os.makedirs(pkiDirpath)
acceptedDirpath = os.path.join(pkiDirpath, 'accepted')
if not os.path.exists(acceptedDirpath):
os.makedirs(acceptedDirpath)
pendingDirpath = os.path.join(pkiDirpath, 'pending')
if not os.path.exists(pendingDirpath):
os.makedirs(pendingDirpath)
rejectedDirpath = os.path.join(pkiDirpath, 'rejected')
if not os.path.exists(rejectedDirpath):
os.makedirs(rejectedDirpath)
localFilepath = os.path.join(pkiDirpath, 'local.key')
if os.path.exists(localFilepath):
mode = os.stat(localFilepath).st_mode
print(mode)
os.chmod(localFilepath, mode | stat.S_IWUSR | stat.S_IRUSR)
mode = os.stat(localFilepath).st_mode
print(mode)
cacheDirpath = os.path.join('/tmp/raet', 'cache', self.role)
if not os.path.exists(cacheDirpath):
os.makedirs(cacheDirpath)
sockDirpath = os.path.join('/tmp/raet', 'sock', self.role)
if not os.path.exists(sockDirpath):
os.makedirs(sockDirpath)
self.opts.value = dict(
id=self.role,
__role=self.role,
ioflo_period=0.1,
ioflo_realtime=True,
ioflo_verbose=2,
interface="",
raet_port=self.raet_port,
transport='raet',
client_acl=dict(),
publisher_acl=dict(),
pki_dir=pkiDirpath,
sock_dir=sockDirpath,
cachedir=cacheDirpath,
open_mode=True,
auto_accept=True,
)
name = "{0}_{1}".format(self.role, self.role)
basedirpath = os.path.abspath(os.path.join(cacheDirpath, 'raet'))
keep = salting.SaltKeep(opts=self.opts.value,
basedirpath=basedirpath,
stackname=name)
keep.clearLocalData()
keep.clearLocalRoleData()
keep.clearAllRemoteData()
keep.clearAllRemoteRoleData()
class TestOptsSetupMaster(TestOptsSetup):
def action(self):
self.role = 'master'
self.raet_port = raeting.RAET_PORT
super(TestOptsSetupMaster, self).action()
class TestOptsSetupMinion(TestOptsSetup):
def action(self):
self.role = 'minion'
self.raet_port = raeting.RAET_TEST_PORT
super(TestOptsSetupMinion, self).action()
def serviceRoads(stacks, duration=1.0):
'''
Utility method to service queues for list of stacks. Call from test method.
'''
start = time.time()
while start + duration > time.time():
for stack in stacks:
stack.serviceAll()
if all([not stack.transactions for stack in stacks]):
console.terse("Service stacks done normally\n")
break
time.sleep(0.05)
for stack in stacks:
console.terse("Stack {0} remotes: {1}\n".format(stack.name, stack.nameRemotes))
console.terse("Service stacks exit\n")
def serviceLanes(stacks, duration=1.0):
'''
Utility method to service queues for list of stacks. Call from test method.
'''
start = time.time()
while start + duration > time.time():
for stack in stacks:
stack.serviceAll()
if all([not stack.txMsgs for stack in stacks]):
console.terse("Service stacks done normally\n")
break
time.sleep(0.05)
for stack in stacks:
console.terse("Stack {0} remotes: {1}\n".format(stack.name, stack.nameRemotes))
console.terse("Service stacks exit\n")
class PresenterTestSetup(ioflo.base.deeding.Deed):
'''
Setup shares for presence tests
'''
Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'),
'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'),
'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'alloweds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.alloweds'),
'ival': odict()},
'aliveds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.aliveds'),
'ival': odict()},
'reapeds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.reapeds'),
'ival': odict()},
'availables': {'ipath': salt.utils.stringutils.to_str(
'.salt.var.presence.availables'),
'ival': set()}}
def action(self):
self.presence_req.value = deque()
self.availables.value = set()
self.alloweds.value = odict()
self.aliveds.value = odict()
self.reapeds.value = odict()
# Create event stack
name = 'event' + nacling.uuid(size=18)
lanename = self.lane_stack.value.local.lanename
sock_dir = self.lane_stack.value.local.dirpath
ryn = 'manor'
console.terse("Create stack: name = {0}, lanename = {1}, sock_dir = {2}\n".
format(name, lanename, sock_dir))
stack = LaneStack(
name=name,
lanename=lanename,
sockdirpath=sock_dir)
stack.addRemote(RemoteYard(stack=stack,
lanename=lanename,
name=ryn,
dirpath=sock_dir))
self.event_stack.value = stack
route = {'dst': (None, ryn, 'presence_req'),
'src': (None, stack.local.name, None)}
msg = {'route': route}
stack.transmit(msg, stack.nameRemotes[ryn].uid)
serviceLanes([stack, self.lane_stack.value])
class PresenterTestCleanup(ioflo.base.deeding.Deed):
'''
Clean up after a test
'''
Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'),
'alloweds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.alloweds'),
'ival': odict()},
'aliveds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.aliveds'),
'ival': odict()},
'reapeds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.reapeds'),
'ival': odict()},
'availables': {'ipath': salt.utils.stringutils.to_str(
'.salt.var.presence.availables'),
'ival': set()}}
def action(self):
self.presence_req.value = deque()
self.availables.value = set()
self.alloweds.value = odict()
self.aliveds.value = odict()
self.reapeds.value = odict()
class TestPresenceAvailable(ioflo.base.deeding.Deed):
Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'),
'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'aliveds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.aliveds'),
'ival': odict()},
'availables': {'ipath': salt.utils.stringutils.to_str(
'.salt.var.presence.availables'),
'ival': set()}}
def action(self):
'''
Test Presenter 'available' request (A1, B*)
'''
console.terse("{0}\n".format(self.action.__doc__))
# Prepare
# add available minions
self.availables.value.add('alpha')
self.availables.value.add('beta')
self.aliveds.value['alpha'] = createStack('1.1.1.1')
self.aliveds.value['beta'] = createStack('1.2.3.4')
# add presence request
testStack = self.event_stack.value
presenceReq = self.presence_req.value
ryn = 'manor'
# general available request format
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'available'}})
# missing 'data', fallback to available
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)}})
# missing 'state' in 'data', fallback to available
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {}})
# requested None state, fallback to available
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': None}})
# requested 'present' state that is alias for available
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'present'}})
class TestPresenceAvailableCheck(ioflo.base.deeding.Deed, DeedTestWrapper):
Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'failure': salt.utils.stringutils.to_str('.meta.failure')}
def action(self):
testStack = self.event_stack.value
self.assertTrue(len(testStack.rxMsgs) == 0)
testStack.serviceAll()
self.assertTrue(len(testStack.rxMsgs) == 5)
tag = tagify('present', 'presence')
while testStack.rxMsgs:
msg, sender = testStack.rxMsgs.popleft()
self.assertTrue(msg == {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'present': {'alpha': '1.1.1.1',
'beta': '1.2.3.4'}}})
class TestPresenceJoined(ioflo.base.deeding.Deed):
Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'),
'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'alloweds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.alloweds'),
'ival': odict()}}
def action(self):
'''
Test Presenter 'joined' request (A2)
'''
console.terse("{0}\n".format(self.action.__doc__))
# Prepare
# add joined minions
self.alloweds.value['alpha'] = createStack('1.1.1.1')
self.alloweds.value['beta'] = createStack('1.2.3.4')
# add presence request
testStack = self.event_stack.value
presenceReq = self.presence_req.value
ryn = 'manor'
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'joined'}})
class TestPresenceJoinedCheck(ioflo.base.deeding.Deed, DeedTestWrapper):
Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'failure': salt.utils.stringutils.to_str('.meta.failure')}
def action(self):
testStack = self.event_stack.value
self.assertTrue(len(testStack.rxMsgs) == 0)
testStack.serviceAll()
self.assertTrue(len(testStack.rxMsgs) == 1)
tag = tagify('present', 'presence')
msg, sender = testStack.rxMsgs.popleft()
self.assertTrue(msg == {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'joined': {'alpha': '1.1.1.1',
'beta': '1.2.3.4'}}})
class TestPresenceAllowed(ioflo.base.deeding.Deed):
Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'),
'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'alloweds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.alloweds'),
'ival': odict()}}
def action(self):
'''
Test Presenter 'allowed' request (A3)
'''
console.terse("{0}\n".format(self.action.__doc__))
# Prepare
# add allowed minions
self.alloweds.value['alpha'] = createStack('1.1.1.1')
self.alloweds.value['beta'] = createStack('1.2.3.4')
# add presence request
testStack = self.event_stack.value
presenceReq = self.presence_req.value
ryn = 'manor'
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'allowed'}})
class TestPresenceAllowedCheck(ioflo.base.deeding.Deed, DeedTestWrapper):
Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'failure': salt.utils.stringutils.to_str('.meta.failure')}
def action(self):
testStack = self.event_stack.value
self.assertTrue(len(testStack.rxMsgs) == 0)
testStack.serviceAll()
self.assertTrue(len(testStack.rxMsgs) == 1)
tag = tagify('present', 'presence')
msg, sender = testStack.rxMsgs.popleft()
self.assertTrue(msg == {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'allowed': {'alpha': '1.1.1.1',
'beta': '1.2.3.4'}}})
class TestPresenceAlived(ioflo.base.deeding.Deed):
Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'),
'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'aliveds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.aliveds'),
'ival': odict()}}
def action(self):
'''
Test Presenter 'alived' request (A4)
'''
console.terse("{0}\n".format(self.action.__doc__))
# Prepare
# add alived minions
self.aliveds.value['alpha'] = createStack('1.1.1.1')
self.aliveds.value['beta'] = createStack('1.2.3.4')
# add presence request
testStack = self.event_stack.value
presenceReq = self.presence_req.value
ryn = 'manor'
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'alived'}})
class TestPresenceAlivedCheck(ioflo.base.deeding.Deed, DeedTestWrapper):
Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'failure': salt.utils.stringutils.to_str('.meta.failure')}
def action(self):
testStack = self.event_stack.value
self.assertTrue(len(testStack.rxMsgs) == 0)
testStack.serviceAll()
self.assertTrue(len(testStack.rxMsgs) == 1)
tag = tagify('present', 'presence')
msg, sender = testStack.rxMsgs.popleft()
self.assertTrue(msg == {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'alived': {'alpha': '1.1.1.1',
'beta': '1.2.3.4'}}})
class TestPresenceReaped(ioflo.base.deeding.Deed):
Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'),
'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'reapeds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.reapeds'),
'ival': odict()}}
def action(self):
'''
Test Presenter 'reaped' request (A5)
'''
console.terse("{0}\n".format(self.action.__doc__))
# Prepare
# add reaped minions
self.reapeds.value['alpha'] = createStack('1.1.1.1')
self.reapeds.value['beta'] = createStack('1.2.3.4')
# add presence request
testStack = self.event_stack.value
presenceReq = self.presence_req.value
ryn = 'manor'
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'reaped'}})
class TestPresenceReapedCheck(ioflo.base.deeding.Deed, DeedTestWrapper):
Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'failure': salt.utils.stringutils.to_str('.meta.failure')}
def action(self):
testStack = self.event_stack.value
self.assertTrue(len(testStack.rxMsgs) == 0)
testStack.serviceAll()
self.assertTrue(len(testStack.rxMsgs) == 1)
tag = tagify('present', 'presence')
msg, sender = testStack.rxMsgs.popleft()
self.assertTrue(msg == {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'reaped': {'alpha': '1.1.1.1',
'beta': '1.2.3.4'}}})
class TestPresenceNoRequest(ioflo.base.deeding.Deed):
Ioinits = {}
def action(self):
'''
Test Presenter with no requests (C1)
'''
console.terse("{0}\n".format(self.action.__doc__))
class TestPresenceNoRequestCheck(ioflo.base.deeding.Deed, DeedTestWrapper):
Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'failure': salt.utils.stringutils.to_str('.meta.failure')}
def action(self):
testStack = self.event_stack.value
self.assertTrue(len(testStack.rxMsgs) == 0)
testStack.serviceAll()
self.assertTrue(len(testStack.rxMsgs) == 0)
class TestPresenceUnknownSrc(ioflo.base.deeding.Deed, DeedTestWrapper):
Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'),
'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'failure': salt.utils.stringutils.to_str('.meta.failure')}
def action(self):
'''
Test Presenter handles request from unknown (disconnected) source (C2)
'''
console.terse("{0}\n".format(self.action.__doc__))
# Prepare
# add presence request
testStack = self.event_stack.value
presenceReq = self.presence_req.value
ryn = 'manor'
name = 'unknown_name'
self.assertTrue(name != testStack.local.name)
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, name, None)}})
class TestPresenceUnknownSrcCheck(ioflo.base.deeding.Deed, DeedTestWrapper):
Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'failure': salt.utils.stringutils.to_str('.meta.failure')}
def action(self):
testStack = self.event_stack.value
self.assertTrue(len(testStack.rxMsgs) == 0)
testStack.serviceAll()
self.assertTrue(len(testStack.rxMsgs) == 0)
class TestPresenceAvailableNoMinions(ioflo.base.deeding.Deed):
Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'),
'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack')}
def action(self):
'''
Test Presenter 'available' request with no minions in the state (D1)
'''
console.terse("{0}\n".format(self.action.__doc__))
# Prepare
# add presence request
testStack = self.event_stack.value
presenceReq = self.presence_req.value
ryn = 'manor'
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'available'}})
class TestPresenceAvailableNoMinionsCheck(ioflo.base.deeding.Deed, DeedTestWrapper):
Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'failure': salt.utils.stringutils.to_str('.meta.failure')}
def action(self):
testStack = self.event_stack.value
self.assertTrue(len(testStack.rxMsgs) == 0)
testStack.serviceAll()
self.assertTrue(len(testStack.rxMsgs) == 1)
tag = tagify('present', 'presence')
while testStack.rxMsgs:
msg, sender = testStack.rxMsgs.popleft()
self.assertTrue(msg == {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'present': {}}})
class TestPresenceAvailableOneMinion(ioflo.base.deeding.Deed):
Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'),
'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'aliveds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.aliveds'),
'ival': odict()},
'availables': {'ipath': salt.utils.stringutils.to_str(
'.salt.var.presence.availables'),
'ival': set()}}
def action(self):
'''
Test Presenter 'available' request with one minions in the state (D2)
'''
console.terse("{0}\n".format(self.action.__doc__))
# Prepare
# add available minions
self.availables.value.add('alpha')
self.aliveds.value['alpha'] = createStack('1.1.1.1')
# add presence request
testStack = self.event_stack.value
presenceReq = self.presence_req.value
ryn = 'manor'
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'available'}})
class TestPresenceAvailableOneMinionCheck(ioflo.base.deeding.Deed, DeedTestWrapper):
Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'failure': salt.utils.stringutils.to_str('.meta.failure')}
def action(self):
testStack = self.event_stack.value
self.assertTrue(len(testStack.rxMsgs) == 0)
testStack.serviceAll()
self.assertTrue(len(testStack.rxMsgs) == 1)
tag = tagify('present', 'presence')
while testStack.rxMsgs:
msg, sender = testStack.rxMsgs.popleft()
self.assertTrue(msg == {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'present': {'alpha': '1.1.1.1'}}})
class TestPresenceAvailableUnknownIp(ioflo.base.deeding.Deed):
Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'),
'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'aliveds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.aliveds'),
'ival': odict()},
'availables': {'ipath': salt.utils.stringutils.to_str(
'.salt.var.presence.availables'),
'ival': set()}}
def action(self):
'''
Test Presenter 'available' request with one minions in the state (D3)
'''
console.terse("{0}\n".format(self.action.__doc__))
# Prepare
# add available minions
self.availables.value.add('alpha')
self.availables.value.add('beta')
self.availables.value.add('gamma')
self.aliveds.value['alpha'] = createStack('1.1.1.1')
self.aliveds.value['delta'] = createStack('1.2.3.4')
# add presence request
testStack = self.event_stack.value
presenceReq = self.presence_req.value
ryn = 'manor'
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'available'}})
class TestPresenceAvailableUnknownIpCheck(ioflo.base.deeding.Deed, DeedTestWrapper):
Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'failure': salt.utils.stringutils.to_str('.meta.failure')}
def action(self):
testStack = self.event_stack.value
self.assertTrue(len(testStack.rxMsgs) == 0)
testStack.serviceAll()
self.assertTrue(len(testStack.rxMsgs) == 1)
tag = tagify('present', 'presence')
while testStack.rxMsgs:
msg, sender = testStack.rxMsgs.popleft()
self.assertTrue(msg == {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'present': {'alpha': '1.1.1.1',
'beta': None,
'gamma': None}}})
class TestPresenceAllowedNoMinions(ioflo.base.deeding.Deed):
Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'),
'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack')}
def action(self):
'''
Test Presenter 'allowed' request with no minions in the state (D4)
'''
console.terse("{0}\n".format(self.action.__doc__))
# Prepare
# add presence request
testStack = self.event_stack.value
presenceReq = self.presence_req.value
ryn = 'manor'
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'allowed'}})
class TestPresenceAllowedNoMinionsCheck(ioflo.base.deeding.Deed, DeedTestWrapper):
Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'failure': salt.utils.stringutils.to_str('.meta.failure')}
def action(self):
testStack = self.event_stack.value
self.assertTrue(len(testStack.rxMsgs) == 0)
testStack.serviceAll()
self.assertTrue(len(testStack.rxMsgs) == 1)
tag = tagify('present', 'presence')
msg, sender = testStack.rxMsgs.popleft()
self.assertTrue(msg == {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'allowed': {}}})
class TestPresenceAllowedOneMinion(ioflo.base.deeding.Deed):
Ioinits = {'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'),
'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'alloweds': {'ipath': salt.utils.stringutils.to_str('.salt.var.presence.alloweds'),
'ival': odict()}}
def action(self):
'''
Test Presenter 'allowed' request with one minion in the state (D5)
'''
console.terse("{0}\n".format(self.action.__doc__))
# Prepare
# add allowed minion
self.alloweds.value['alpha'] = createStack('1.1.1.1')
# add presence request
testStack = self.event_stack.value
presenceReq = self.presence_req.value
ryn = 'manor'
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'allowed'}})
class TestPresenceAllowedOneMinionCheck(ioflo.base.deeding.Deed, DeedTestWrapper):
Ioinits = {'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack'),
'failure': salt.utils.stringutils.to_str('.meta.failure')}
def action(self):
testStack = self.event_stack.value
self.assertTrue(len(testStack.rxMsgs) == 0)
testStack.serviceAll()
self.assertTrue(len(testStack.rxMsgs) == 1)
tag = tagify('present', 'presence')
msg, sender = testStack.rxMsgs.popleft()
self.assertTrue(msg == {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'allowed': {'alpha': '1.1.1.1'}}})
class StatsMasterTestSetup(ioflo.base.deeding.Deed):
'''
Setup shares for stats tests
'''
Ioinits = {'stats_req': salt.utils.stringutils.to_str('.salt.stats.event_req'),
'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'),
'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'),
'event_stack': salt.utils.stringutils.to_str('.salt.test.lane.stack')}
def action(self):
self.stats_req.value = deque()
# Create event stack
name = 'event' + nacling.uuid(size=18)
lanename = self.lane_stack.value.local.lanename
sock_dir = self.lane_stack.value.local.dirpath
ryn = 'manor'
console.terse("Create stack: name = {0}, lanename = {1}, sock_dir = {2}\n".
format(name, lanename, sock_dir))
stack = LaneStack(
name=name,
lanename=lanename,
sockdirpath=sock_dir)
stack.addRemote(RemoteYard(stack=stack,
lanename=lanename,
name=ryn,
dirpath=sock_dir))
self.event_stack.value = stack
route = {'dst': (None, ryn, 'stats_req'),
'src': (None, stack.local.name, None)}
msg = {'route': route}
stack.transmit(msg, stack.nameRemotes[ryn].uid)
serviceLanes([stack, self.lane_stack.value])
class StatsMinionTestSetup(ioflo.base.deeding.Deed):
'''
Setup shares for stats tests
'''
Ioinits = {'stats_req': salt.utils.stringutils.to_str('.salt.stats.event_req'),
'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'),
'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack'),
'event_stack': salt.utils.stringutils.to_str('.salt.test.road.stack')}
def action(self):
self.stats_req.value = deque()
minionStack = self.road_stack.value
# Create Master Stack
self.store.stamp = 0.0
masterStack = RoadStack(store=self.store,
name='master',
ha=('', raeting.RAET_PORT),
role='master',
main=True,
cleanremote=True,
period=3.0,
offset=0.5)
self.event_stack.value = masterStack
minionRemoteMaster = RemoteEstate(stack=minionStack,
fuid=0,
sid=0,
ha=masterStack.local.ha)
minionStack.addRemote(minionRemoteMaster)
# Make life easier
masterStack.keep.auto = raeting.AutoMode.always.value
minionStack.keep.auto = raeting.AutoMode.always.value
minionStack.join(minionRemoteMaster.uid)
serviceRoads([minionStack, masterStack])
minionStack.allow(minionRemoteMaster.uid)
serviceRoads([minionStack, masterStack])

118
salt/daemons/test/plan/box1.flo
View file

@ -1,118 +0,0 @@
# Test for presence functionality
house presence
framer presenter be active first setup
frame setup
do test opts setup master
do salt raet manor lane setup per inode ".salt.lane.manor"
do presenter test setup
go next
# A1, B*
frame test_presence_available
do test presence available
do salt raet presenter
do test presence available check
do presenter test cleanup
go next
# A2
frame test_presence_joined
do test presence joined
do salt raet presenter
do test presence joined check
do presenter test cleanup
go next
# A3
frame test_presence_allowed
do test presence allowed
do salt raet presenter
do test presence allowed check
do presenter test cleanup
go next
# A4
frame test_presence_alived
do test presence alived
do salt raet presenter
do test presence alived check
do presenter test cleanup
go next
# A5
frame test_presence_reaped
do test presence reaped
do salt raet presenter
do test presence reaped check
do presenter test cleanup
go next
# C1
frame test_presence_no_request
do test presence no request
do salt raet presenter
do test presence no request check
do presenter test cleanup
go next
# C2
frame test_presence_unknown_src
do test presence unknown src
do salt raet presenter
do test presence unknown src check
do presenter test cleanup
go next
# D1
frame test_presence_available_no_minions
do test presence available no minions
do salt raet presenter
do test presence available no minions check
do presenter test cleanup
go next
# D2
frame test_presence_available_one_minion
do test presence available one minion
do salt raet presenter
do test presence available one minion check
do presenter test cleanup
go next
# D3
frame test_presence_available_unknown_ip
do test presence available unknown ip
do salt raet presenter
do test presence available unknown ip check
do presenter test cleanup
go next
# D4
frame test_presence_allowed_no_minions
do test presence allowed no minions
do salt raet presenter
do test presence allowed no minions
do presenter test cleanup
go next
# D4
frame test_presence_allowed_one_minion
do test presence allowed one minion
do salt raet presenter
do test presence allowed one minion
do presenter test cleanup
go next
frame done
go abort if meta.failure
go success
frame success
print "Success"
bid stop all
frame abort
print "Failure"
bid stop all

84
salt/daemons/test/test_master.py
View file

@ -1,84 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
Runs minion floscript
'''
from __future__ import absolute_import, print_function, unicode_literals
# pylint: skip-file
import os
import stat
from ioflo.aid.odicting import odict
from ioflo.base.consoling import getConsole
console = getConsole()
import salt.daemons.flo
def test():
""" Execute run.start """
pkiDirpath = os.path.join('/tmp', 'raet', 'testo', 'master', 'pki')
if not os.path.exists(pkiDirpath):
os.makedirs(pkiDirpath)
keyDirpath = os.path.join('/tmp', 'raet', 'testo', 'key')
if not os.path.exists(keyDirpath):
os.makedirs(keyDirpath)
acceptedDirpath = os.path.join(pkiDirpath, 'accepted')
if not os.path.exists(acceptedDirpath):
os.makedirs(acceptedDirpath)
pendingDirpath = os.path.join(pkiDirpath, 'pending')
if not os.path.exists(pendingDirpath):
os.makedirs(pendingDirpath)
rejectedDirpath = os.path.join(pkiDirpath, 'rejected')
if not os.path.exists(rejectedDirpath):
os.makedirs(rejectedDirpath)
localFilepath = os.path.join(pkiDirpath, 'local.key')
if os.path.exists(localFilepath):
mode = os.stat(localFilepath).st_mode
print(mode)
os.chmod(localFilepath, mode | stat.S_IWUSR | stat.S_IRUSR)
mode = os.stat(localFilepath).st_mode
print(mode)
cacheDirpath = os.path.join('/tmp/raet', 'cache', 'master')
if not os.path.exists(cacheDirpath):
os.makedirs(cacheDirpath)
sockDirpath = os.path.join('/tmp/raet', 'sock', 'master')
if not os.path.exists(sockDirpath):
os.makedirs(sockDirpath)
filepath = 'master.flo'
opts = dict(
id='master',
__role='master',
ioflo_period=0.1,
ioflo_realtime=True,
master_floscript=filepath,
ioflo_verbose=2,
interface="",
raet_port=7530,
transport='raet',
client_acl=dict(),
publisher_acl=dict(),
pki_dir=pkiDirpath,
key_dir=keyDirpath,
sock_dir=sockDirpath,
cachedir=cacheDirpath,
open_mode=True,
auto_accept=True,
client_acl_verify=True,
)
master = salt.daemons.flo.IofloMaster(opts=opts)
master.start(behaviors=['raet.flo.behaving'])
if __name__ == '__main__':
console.reinit(verbosity=console.Wordage.concise)
test()

84
salt/daemons/test/test_minion.py
View file

@ -1,84 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
Runs minion floscript
'''
from __future__ import absolute_import, print_function, unicode_literals
import os
import stat
from ioflo.base.consoling import getConsole
console = getConsole()
import salt.daemons.flo
FLO_DIR_PATH = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'flo'
)
def test():
""" Execute run.start """
pkiDirpath = os.path.join('/tmp', 'raet', 'testo', 'minion', 'pki')
if not os.path.exists(pkiDirpath):
os.makedirs(pkiDirpath)
acceptedDirpath = os.path.join(pkiDirpath, 'accepted')
if not os.path.exists(acceptedDirpath):
os.makedirs(acceptedDirpath)
pendingDirpath = os.path.join(pkiDirpath, 'pending')
if not os.path.exists(pendingDirpath):
os.makedirs(pendingDirpath)
rejectedDirpath = os.path.join(pkiDirpath, 'rejected')
if not os.path.exists(rejectedDirpath):
os.makedirs(rejectedDirpath)
localFilepath = os.path.join(pkiDirpath, 'local.key')
if os.path.exists(localFilepath):
mode = os.stat(localFilepath).st_mode
print(mode)
os.chmod(localFilepath, mode | stat.S_IWUSR | stat.S_IRUSR)
mode = os.stat(localFilepath).st_mode
print(mode)
cacheDirpath = os.path.join('/tmp/raet', 'cache', 'minion')
if not os.path.exists(cacheDirpath):
os.makedirs(cacheDirpath)
sockDirpath = os.path.join('/tmp/raet', 'sock', 'minion')
if not os.path.exists(sockDirpath):
os.makedirs(sockDirpath)
#filepath = os.path.join(FLO_DIR_PATH, 'minion.flo')
filepath = 'minion.flo'
opts = dict(
id="minion",
__role='minion',
ioflo_period=0.1,
ioflo_realtime=True,
minion_floscript=filepath,
ioflo_verbose=2,
interface="",
raet_port=7531,
master_port=7530,
master='127.0.0.1',
transport='raet',
client_acl=dict(),
publisher_acl=dict(),
pki_dir=pkiDirpath,
sock_dir=sockDirpath,
cachedir=cacheDirpath,
open_mode=True,
auto_accept=True)
minion = salt.daemons.flo.IofloMinion(opts=opts)
minion.start(behaviors=['raet.flo.behaving'])
if __name__ == '__main__':
console.reinit(verbosity=console.Wordage.concise)
test()

355
salt/daemons/test/test_multimaster.py
View file

@ -1,355 +0,0 @@
# -*- coding: utf-8 -*-
'''
Tests of utilities that support multiple masters in Salt Raet
'''
from __future__ import absolute_import, print_function, unicode_literals
import sys
from salt.ext.six.moves import map
# pylint: disable=blacklisted-import
if sys.version_info < (2, 7):
import unittest2 as unittest
else:
import unittest
# pylint: enable=blacklisted-import
from ioflo.aid.timing import StoreTimer
from ioflo.base import storing
from ioflo.base.consoling import getConsole
console = getConsole()
from salt.daemons import parse_hostname, extract_masters
def setUpModule():
console.reinit(verbosity=console.Wordage.concise)
def tearDownModule():
pass
class BasicTestCase(unittest.TestCase): # pylint: disable=moved-test-case-class
def setUp(self):
self.store = storing.Store(stamp=0.0)
self.timer = StoreTimer(store=self.store, duration=1.0)
self.port = 4506
self.opts = dict(master_port=self.port)
def tearDown(self):
pass
def testParseHostname(self):
'''
Test parsing hostname provided according to syntax for opts['master']
'''
console.terse("{0}\n".format(self.testParseHostname.__doc__))
self.assertEquals(parse_hostname('localhost', self.port),
('localhost', 4506))
self.assertEquals(parse_hostname('127.0.0.1', self.port),
('127.0.0.1', 4506))
self.assertEquals(parse_hostname('10.0.2.100', self.port),
('10.0.2.100', 4506))
self.assertEquals(parse_hostname('me.example.com', self.port),
('me.example.com', 4506))
self.assertEquals(parse_hostname(
'1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa',
self.port),
('1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa',
4506))
self.assertEquals(parse_hostname('fe80::1%lo0', self.port),
('fe80::1%lo0', 4506))
self.assertEquals(parse_hostname(' localhost ', self.port),
('localhost', 4506))
self.assertEquals(parse_hostname(' 127.0.0.1 ', self.port),
('127.0.0.1', 4506))
self.assertEquals(parse_hostname(' 10.0.2.100 ', self.port),
('10.0.2.100', 4506))
self.assertEquals(parse_hostname(' me.example.com ', self.port),
('me.example.com', 4506))
self.assertEquals(parse_hostname(
' 1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa ',
self.port),
('1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa',
4506))
self.assertEquals(parse_hostname(' fe80::1%lo0 ', self.port),
('fe80::1%lo0', 4506))
self.assertEquals(parse_hostname('localhost 4510', self.port),
('localhost', 4510))
self.assertEquals(parse_hostname('127.0.0.1 4510', self.port),
('127.0.0.1', 4510))
self.assertEquals(parse_hostname('10.0.2.100 4510', self.port),
('10.0.2.100', 4510))
self.assertEquals(parse_hostname('me.example.com 4510', self.port),
('me.example.com', 4510))
self.assertEquals(parse_hostname(
'1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa 4510',
self.port),
('1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa',
4510))
self.assertEquals(parse_hostname('fe80::1%lo0 4510', self.port),
('fe80::1%lo0', 4510))
self.assertEquals(parse_hostname(' localhost 4510 ', self.port),
('localhost', 4510))
self.assertEquals(parse_hostname(' 127.0.0.1 4510 ', self.port),
('127.0.0.1', 4510))
self.assertEquals(parse_hostname(' 10.0.2.100 4510 ', self.port),
('10.0.2.100', 4510))
self.assertEquals(parse_hostname(' me.example.com 4510 ', self.port),
('me.example.com', 4510))
self.assertEquals(parse_hostname(
' 1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa 4510 ',
self.port),
('1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa',
4510))
self.assertEquals(parse_hostname(' fe80::1%lo0 4510 ', self.port),
('fe80::1%lo0', 4510))
self.assertEquals(parse_hostname('localhost abcde', self.port), None)
self.assertEquals(parse_hostname('127.0.0.1 a4510', self.port), None)
self.assertEquals(parse_hostname(list([1, 2, 3]), self.port), None)
self.assertEquals(parse_hostname(list(), self.port), None)
self.assertEquals(parse_hostname(dict(a=1), self.port), None)
self.assertEquals(parse_hostname(dict(), self.port), None)
self.assertEquals(parse_hostname(4510, self.port), None)
self.assertEquals(parse_hostname(('localhost', 4510), self.port), None)
self.assertEquals(parse_hostname('localhost:4510', self.port),
('localhost', 4510))
self.assertEquals(parse_hostname('127.0.0.1:4510', self.port),
('127.0.0.1', 4510))
self.assertEquals(parse_hostname('10.0.2.100:4510', self.port),
('10.0.2.100', 4510))
self.assertEquals(parse_hostname('me.example.com:4510', self.port),
('me.example.com', 4510))
self.assertEquals(parse_hostname(
'1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa:4510',
self.port),
('1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa',
4510))
self.assertEquals(parse_hostname('fe80::1%lo0:4510', self.port),
('fe80::1%lo0:4510', 4506))
self.assertEquals(parse_hostname('localhost::4510', self.port),
('localhost::4510', 4506))
def testExtractMastersSingle(self):
'''
Test extracting from master provided according to syntax for opts['master']
'''
console.terse("{0}\n".format(self.testExtractMastersSingle.__doc__))
master = 'localhost'
self.opts.update(master=master)
self.assertEquals(extract_masters(self.opts),
[
dict(external=('localhost', 4506),
internal=None),
])
master = '127.0.0.1'
self.opts.update(master=master)
self.assertEquals(extract_masters(self.opts),
[
dict(external=('127.0.0.1', 4506),
internal=None),
])
master = 'localhost 4510'
self.opts.update(master=master)
self.assertEquals(extract_masters(self.opts),
[
dict(external=('localhost', 4510),
internal=None),
])
master = '127.0.0.1 4510'
self.opts.update(master=master)
self.assertEquals(extract_masters(self.opts),
[
dict(external=('127.0.0.1', 4510),
internal=None),
])
master = '10.0.2.23'
self.opts.update(master=master)
self.assertEquals(extract_masters(self.opts),
[
dict(external=('10.0.2.23', 4506),
internal=None),
])
master = 'me.example.com'
self.opts.update(master=master)
self.assertEquals(extract_masters(self.opts),
[
dict(external=('me.example.com', 4506),
internal=None),
])
master = '10.0.2.23 4510'
self.opts.update(master=master)
self.assertEquals(extract_masters(self.opts),
[
dict(external=('10.0.2.23', 4510),
internal=None),
])
master = 'me.example.com 4510'
self.opts.update(master=master)
self.assertEquals(extract_masters(self.opts),
[
dict(external=('me.example.com', 4510),
internal=None),
])
master = dict(external='10.0.2.23 4510')
self.opts.update(master=master)
self.assertEquals(extract_masters(self.opts),
[
dict(external=('10.0.2.23', 4510),
internal=None),
])
master = dict(external='10.0.2.23 4510', internal='')
self.opts.update(master=master)
self.assertEquals(extract_masters(self.opts),
[
dict(external=('10.0.2.23', 4510),
internal=None),
])
master = dict(internal='10.0.2.23 4510')
self.opts.update(master=master)
self.assertEquals(extract_masters(self.opts), [])
def testExtractMastersMultiple(self):
'''
Test extracting from master provided according to syntax for opts['master']
'''
console.terse("{0}\n".format(self.testExtractMastersMultiple.__doc__))
master = [
'localhost',
'10.0.2.23',
'me.example.com'
]
self.opts.update(master=master)
self.assertEquals(extract_masters(self.opts),
[
{
'external': ('localhost', 4506),
'internal': None
},
{
'external': ('10.0.2.23', 4506),
'internal': None
},
{
'external': ('me.example.com', 4506),
'internal': None
},
])
master = [
'localhost 4510',
'10.0.2.23 4510',
'me.example.com 4510'
]
self.opts.update(master=master)
self.assertEquals(extract_masters(self.opts),
[
{
'external': ('localhost', 4510),
'internal': None
},
{
'external': ('10.0.2.23', 4510),
'internal': None
},
{
'external': ('me.example.com', 4510),
'internal': None
},
])
master = [
{
'external': 'localhost 4510',
'internal': '',
},
{
'external': 'me.example.com 4510',
'internal': '10.0.2.23 4510',
},
{
'external': 'you.example.com 4509',
}
]
self.opts.update(master=master)
self.assertEquals(extract_masters(self.opts),
[
{
'external': ('localhost', 4510),
'internal': None
},
{
'external': ('me.example.com', 4510),
'internal': ('10.0.2.23', 4510)
},
{
'external': ('you.example.com', 4509),
'internal': None
},
])
def runOne(test):
'''
Unittest Runner
'''
test = BasicTestCase(test)
suite = unittest.TestSuite([test])
unittest.TextTestRunner(verbosity=2).run(suite)
def runSome():
'''
Unittest runner
'''
tests = []
names = [
'testParseHostname',
'testExtractMastersSingle',
'testExtractMastersMultiple',
]
tests.extend(list(list(map(BasicTestCase, names))))
suite = unittest.TestSuite(tests)
unittest.TextTestRunner(verbosity=2).run(suite)
def runAll():
'''
Unittest runner
'''
suite = unittest.TestSuite()
suite.addTest(unittest.TestLoader().loadTestsFromTestCase(BasicTestCase))
unittest.TextTestRunner(verbosity=2).run(suite)
if __name__ == '__main__' and __package__ is None:
#console.reinit(verbosity=console.Wordage.concise)
runAll() # run all unittests
#runSome() # only run some
#runOne('testParseHostname')

72
salt/daemons/test/test_plans.py
View file

@ -1,72 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
Runs all the example FloScripts
'''
# pylint: disable=3rd-party-module-not-gated
# Import Python Libs
from __future__ import absolute_import, print_function, unicode_literals
import os
# Import 3rd-party libs
import ioflo.app.run
from ioflo.base.consoling import getConsole
console = getConsole()
PLAN_DIR_PATH = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'plan'
)
def getPlanFiles(planDirPath=PLAN_DIR_PATH):
planFiles = []
for fname in os.listdir(os.path.abspath(planDirPath)):
root, ext = os.path.splitext(fname)
if ext != '.flo' or root.startswith('__'):
continue
planFiles.append(os.path.abspath(os.path.join(planDirPath, fname)))
return planFiles
def main():
'''
Run example scripts
'''
console.concise('Test started')
behaviors = ['salt.daemons.flo', 'salt.daemons.test.plan']
failedCount = 0
plans = getPlanFiles()
for plan in plans:
name, ext = os.path.splitext(os.path.basename(plan))
skeddar = ioflo.app.run.run(name=name,
filepath=plan,
behaviors=behaviors,
period=0.0625,
verbose=1,
real=False,)
print('Plan {0}\n Skeddar {1}\n'.format(plan, skeddar.name))
failed = False
for house in skeddar.houses:
failure = house.metas['failure'].value
if failure:
failed = True
print('**** Failed in House = {0}. '
'Failure = {1}.\n'.format(house.name, failure))
else:
print('**** Succeeded in House = {0}.\n'.format(house.name))
if failed:
failedCount += 1
print('{0} failed out of {1}.\n'.format(failedCount, len(plans)))
if __name__ == '__main__':
console.reinit(verbosity=console.Wordage.profuse)
main()

740
salt/daemons/test/test_presence.py
View file

@ -1,740 +0,0 @@
# -*- coding: utf-8 -*-
'''
Raet Ioflo Behavior Unittests
'''
from __future__ import absolute_import, print_function, unicode_literals
import sys
from salt.ext.six.moves import map
import importlib
# pylint: disable=blacklisted-import
if sys.version_info < (2, 7):
import unittest2 as unittest
else:
import unittest
# pylint: enable=blacklisted-import
from ioflo.base.consoling import getConsole
console = getConsole()
from ioflo.aid.odicting import odict
from ioflo.test import testing
from raet.lane.stacking import LaneStack
from raet.stacking import Stack
import salt.utils.stringutils
from salt.utils.event import tagify
def setUpModule():
console.reinit(verbosity=console.Wordage.concise)
def tearDownModule():
pass
class PresenterTestCase(testing.FrameIofloTestCase):
'''
Test case for Salt Raet Presenter deed
'''
def setUp(self):
'''
Call super if override so House Framer and Frame are setup correctly
'''
behaviors = ['salt.daemons.flo', 'salt.daemons.test.plan']
for behavior in behaviors:
mod = importlib.import_module(behavior)
super(PresenterTestCase, self).setUp()
def tearDown(self):
'''
Call super if override so House Framer and Frame are torn down correctly
'''
super(PresenterTestCase, self).tearDown()
def addPresenceInfo(self, stateGrp, name, ip, port):
self.assertIn(stateGrp, ('alloweds', 'aliveds', 'reapeds'))
group = self.store.fetch('.salt.var.presence.{0}'.format(stateGrp))
if group.value is None:
group.value = odict()
remote = Stack()
remote.ha = (ip, port)
group.value[name] = remote
def addAvailable(self, name):
availables = self.store.fetch('.salt.var.presence.availables')
if availables.value is None:
availables.value = set()
availables.value.add(name)
def testContextSetup(self):
'''
Test the context setup procedure used in all the consequence tests works as expected
This test intended to avoid some checks in other tests
'''
console.terse("{0}\n".format(self.testContextSetup.__doc__))
act = self.addEnterDeed("TestOptsSetupMaster")
self.assertIn(act, self.frame.enacts)
self.assertEqual(act.actor, "TestOptsSetupMaster")
act = self.addEnterDeed("SaltRaetManorLaneSetup")
self.assertIn(act, self.frame.enacts)
self.assertEqual(act.actor, "SaltRaetManorLaneSetup")
act = self.addEnterDeed("PresenterTestSetup")
self.assertIn(act, self.frame.enacts)
self.assertEqual(act.actor, "PresenterTestSetup")
act = self.addRecurDeed("SaltRaetPresenter")
self.assertIn(act, self.frame.reacts)
self.assertEqual(act.actor, "SaltRaetPresenter")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
self.assertDictEqual(
act.actor.Ioinits,
{'opts': salt.utils.stringutils.to_str('.salt.opts'),
'presence_req': salt.utils.stringutils.to_str('.salt.presence.event_req'),
'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'),
'alloweds': salt.utils.stringutils.to_str('.salt.var.presence.alloweds'),
'aliveds': salt.utils.stringutils.to_str('.salt.var.presence.aliveds'),
'reapeds': salt.utils.stringutils.to_str('.salt.var.presence.reapeds'),
'availables': salt.utils.stringutils.to_str('.salt.var.presence.availables')})
self.assertTrue(hasattr(act.actor, 'opts'))
self.assertTrue(hasattr(act.actor, 'presence_req'))
self.assertTrue(hasattr(act.actor, 'lane_stack'))
self.assertTrue(hasattr(act.actor, 'alloweds'))
self.assertTrue(hasattr(act.actor, 'aliveds'))
self.assertTrue(hasattr(act.actor, 'reapeds'))
self.assertTrue(hasattr(act.actor, 'availables'))
self.assertIsInstance(act.actor.lane_stack.value, LaneStack)
self.frame.recur()
# Close active stacks servers
act.actor.lane_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def testPresenceAvailable(self):
'''
Test Presenter 'available' request (A1, B*)
'''
console.terse("{0}\n".format(self.testPresenceAvailable.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("PresenterTestSetup")
act = self.addRecurDeed("SaltRaetPresenter")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add available minions
self.addAvailable('alpha')
self.addAvailable('beta')
self.addPresenceInfo('aliveds', 'alpha', '1.1.1.1', '1234')
self.addPresenceInfo('aliveds', 'beta', '1.2.3.4', '1234')
# add presence request
testStack = self.store.fetch('.salt.test.lane.stack').value
presenceReq = self.store.fetch('.salt.presence.event_req').value
ryn = 'manor'
# general available request format
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'available'}})
# missing 'data', fallback to available
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)}})
# missing 'state' in 'data', fallback to available
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {}})
# requested None state, fallback to available
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': None}})
# requested 'present' state that is alias for available
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'present'}})
# Test
# process 5 requests at once
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 5)
tag = tagify('present', 'presence')
while testStack.rxMsgs:
msg, sender = testStack.rxMsgs.popleft()
self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'present': {'alpha': '1.1.1.1',
'beta': '1.2.3.4'}}})
# Close active stacks servers
act.actor.lane_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def testPresenceJoined(self):
'''
Test Presenter 'joined' request (A2)
'''
console.terse("{0}\n".format(self.testPresenceJoined.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("PresenterTestSetup")
act = self.addRecurDeed("SaltRaetPresenter")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add joined minions
# NOTE: for now alloweds are threaded as joineds
self.addPresenceInfo('alloweds', 'alpha', '1.1.1.1', '1234')
self.addPresenceInfo('alloweds', 'beta', '1.2.3.4', '1234')
# add presence request
testStack = self.store.fetch('.salt.test.lane.stack').value
presenceReq = self.store.fetch('.salt.presence.event_req').value
ryn = 'manor'
msg = {'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'joined'}}
presenceReq.append(msg)
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 1)
tag = tagify('present', 'presence')
msg, sender = testStack.rxMsgs.popleft()
self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'joined': {'alpha': '1.1.1.1',
'beta': '1.2.3.4'}}})
# Close active stacks servers
act.actor.lane_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def testPresenceAllowed(self):
'''
Test Presenter 'allowed' request (A3)
'''
console.terse("{0}\n".format(self.testPresenceAllowed.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("PresenterTestSetup")
act = self.addRecurDeed("SaltRaetPresenter")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add allowed minions
self.addPresenceInfo('alloweds', 'alpha', '1.1.1.1', '1234')
self.addPresenceInfo('alloweds', 'beta', '1.2.3.4', '1234')
# add presence request
testStack = self.store.fetch('.salt.test.lane.stack').value
presenceReq = self.store.fetch('.salt.presence.event_req').value
ryn = 'manor'
msg = {'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'allowed'}}
presenceReq.append(msg)
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 1)
tag = tagify('present', 'presence')
msg, sender = testStack.rxMsgs.popleft()
self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'allowed': {'alpha': '1.1.1.1',
'beta': '1.2.3.4'}}})
# Close active stacks servers
act.actor.lane_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def testPresenceAlived(self):
'''
Test Presenter 'alived' request (A4)
'''
console.terse("{0}\n".format(self.testPresenceAlived.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("PresenterTestSetup")
act = self.addRecurDeed("SaltRaetPresenter")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add alived minions
self.addPresenceInfo('aliveds', 'alpha', '1.1.1.1', '1234')
self.addPresenceInfo('aliveds', 'beta', '1.2.3.4', '1234')
# add presence request
testStack = self.store.fetch('.salt.test.lane.stack').value
presenceReq = self.store.fetch('.salt.presence.event_req').value
ryn = 'manor'
msg = {'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'alived'}}
presenceReq.append(msg)
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 1)
tag = tagify('present', 'presence')
msg, sender = testStack.rxMsgs.popleft()
self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'alived': {'alpha': '1.1.1.1',
'beta': '1.2.3.4'}}})
# Close active stacks servers
act.actor.lane_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def testPresenceReaped(self):
'''
Test Presenter 'reaped' request (A5)
'''
console.terse("{0}\n".format(self.testPresenceReaped.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("PresenterTestSetup")
act = self.addRecurDeed("SaltRaetPresenter")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add reaped minions
self.addPresenceInfo('reapeds', 'alpha', '1.1.1.1', '1234')
self.addPresenceInfo('reapeds', 'beta', '1.2.3.4', '1234')
# add presence request
testStack = self.store.fetch('.salt.test.lane.stack').value
presenceReq = self.store.fetch('.salt.presence.event_req').value
ryn = 'manor'
msg = {'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'reaped'}}
presenceReq.append(msg)
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 1)
tag = tagify('present', 'presence')
msg, sender = testStack.rxMsgs.popleft()
self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'reaped': {'alpha': '1.1.1.1',
'beta': '1.2.3.4'}}})
# Close active stacks servers
act.actor.lane_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def testPresenceNoRequest(self):
'''
Test Presenter with no requests (C1)
'''
console.terse("{0}\n".format(self.testPresenceNoRequest.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("PresenterTestSetup")
act = self.addRecurDeed("SaltRaetPresenter")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Test
self.frame.recur() # run in frame
# Check
testStack = self.store.fetch('.salt.test.lane.stack').value
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 0)
# Close active stacks servers
act.actor.lane_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def testPresenceUnknownSrc(self):
'''
Test Presenter handles request from unknown (disconnected) source (C2)
'''
console.terse("{0}\n".format(self.testPresenceUnknownSrc.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("PresenterTestSetup")
act = self.addRecurDeed("SaltRaetPresenter")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add presence request
testStack = self.store.fetch('.salt.test.lane.stack').value
presenceReq = self.store.fetch('.salt.presence.event_req').value
ryn = 'manor'
name = 'unknown_name'
self.assertNotEqual(name, testStack.local.name)
msg = {'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, name, None)}}
presenceReq.append(msg)
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 0)
# Close active stacks servers
act.actor.lane_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def testPresenceAvailableNoMinions(self):
'''
Test Presenter 'available' request with no minions in the state (D1)
'''
console.terse("{0}\n".format(self.testPresenceAvailableNoMinions.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("PresenterTestSetup")
act = self.addRecurDeed("SaltRaetPresenter")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add presence request
testStack = self.store.fetch('.salt.test.lane.stack').value
presenceReq = self.store.fetch('.salt.presence.event_req').value
ryn = 'manor'
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'available'}})
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 1)
tag = tagify('present', 'presence')
msg, sender = testStack.rxMsgs.popleft()
self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'present': {}}})
# Close active stacks servers
act.actor.lane_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def testPresenceAvailableOneMinion(self):
'''
Test Presenter 'available' request with one minion in the state (D2)
'''
console.terse("{0}\n".format(self.testPresenceAvailableOneMinion.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("PresenterTestSetup")
act = self.addRecurDeed("SaltRaetPresenter")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add available minions
self.addAvailable('alpha')
self.addPresenceInfo('aliveds', 'alpha', '1.1.1.1', '1234')
# add presence request
testStack = self.store.fetch('.salt.test.lane.stack').value
presenceReq = self.store.fetch('.salt.presence.event_req').value
ryn = 'manor'
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'available'}})
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 1)
tag = tagify('present', 'presence')
msg, sender = testStack.rxMsgs.popleft()
self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'present': {'alpha': '1.1.1.1'}}})
# Close active stacks servers
act.actor.lane_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def testPresenceAvailableSomeIpUnknown(self):
'''
Test Presenter 'available' request with some minion addresses aren't known (D3)
'''
console.terse("{0}\n".format(self.testPresenceAvailableSomeIpUnknown.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("PresenterTestSetup")
act = self.addRecurDeed("SaltRaetPresenter")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add available minions
self.addAvailable('alpha')
self.addAvailable('beta')
self.addAvailable('gamma')
self.addPresenceInfo('aliveds', 'alpha', '1.1.1.1', '1234')
self.addPresenceInfo('aliveds', 'delta', '1.2.3.4', '1234')
# add presence request
testStack = self.store.fetch('.salt.test.lane.stack').value
presenceReq = self.store.fetch('.salt.presence.event_req').value
ryn = 'manor'
presenceReq.append({'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'available'}})
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 1)
tag = tagify('present', 'presence')
msg, sender = testStack.rxMsgs.popleft()
self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'present': {'alpha': '1.1.1.1',
'beta': None,
'gamma': None}}})
# Close active stacks servers
act.actor.lane_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def testPresenceAllowedNoMinions(self):
'''
Test Presenter 'allowed' request with no minions in the state (D4)
'''
console.terse("{0}\n".format(self.testPresenceAllowedNoMinions.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("PresenterTestSetup")
act = self.addRecurDeed("SaltRaetPresenter")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add presence request
testStack = self.store.fetch('.salt.test.lane.stack').value
presenceReq = self.store.fetch('.salt.presence.event_req').value
ryn = 'manor'
msg = {'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'allowed'}}
presenceReq.append(msg)
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 1)
tag = tagify('present', 'presence')
msg, sender = testStack.rxMsgs.popleft()
self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'allowed': {}}})
# Close active stacks servers
act.actor.lane_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def testPresenceAllowedOneMinion(self):
'''
Test Presenter 'allowed' request with one minion in the state (D5)
'''
console.terse("{0}\n".format(self.testPresenceAllowedOneMinion.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("PresenterTestSetup")
act = self.addRecurDeed("SaltRaetPresenter")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add allowed minions
self.addPresenceInfo('alloweds', 'alpha', '1.1.1.1', '1234')
# add presence request
testStack = self.store.fetch('.salt.test.lane.stack').value
presenceReq = self.store.fetch('.salt.presence.event_req').value
ryn = 'manor'
msg = {'route': {'dst': (None, ryn, 'presence_req'),
'src': (None, testStack.local.name, None)},
'data': {'state': 'allowed'}}
presenceReq.append(msg)
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 1)
tag = tagify('present', 'presence')
msg, sender = testStack.rxMsgs.popleft()
self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'allowed': {'alpha': '1.1.1.1'}}})
# Close active stacks servers
act.actor.lane_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def runOne(test):
'''
Unittest Runner
'''
test = PresenterTestCase(test)
suite = unittest.TestSuite([test])
unittest.TextTestRunner(verbosity=2).run(suite)
def runSome():
'''
Unittest runner
'''
tests = []
names = [
'testContextSetup',
'testPresenceAvailable',
'testPresenceJoined',
'testPresenceAllowed',
'testPresenceAlived',
'testPresenceReaped',
'testPresenceNoRequest',
'testPresenceUnknownSrc',
'testPresenceAvailableNoMinions',
'testPresenceAvailableOneMinion',
'testPresenceAvailableSomeIpUnknown',
'testPresenceAllowedNoMinions',
'testPresenceAllowedOneMinion',
]
tests.extend(list(map(PresenterTestCase, names)))
suite = unittest.TestSuite(tests)
unittest.TextTestRunner(verbosity=2).run(suite)
def runAll():
'''
Unittest runner
'''
suite = unittest.TestSuite()
suite.addTest(unittest.TestLoader().loadTestsFromTestCase(PresenterTestCase))
unittest.TextTestRunner(verbosity=2).run(suite)
if __name__ == '__main__' and __package__ is None:
# console.reinit(verbosity=console.Wordage.concise)
runAll() # run all unittests
# runSome() #only run some
# runOne('testPresenceAvailable')

442
salt/daemons/test/test_raetkey.py
View file

@ -1,442 +0,0 @@
# -*- coding: utf-8 -*-
'''
Tests to try out salt key.RaetKey Potentially ephemeral
'''
from __future__ import absolute_import, print_function, unicode_literals
# pylint: skip-file
# pylint: disable=C0103
import sys
import salt.utils.stringutils
from salt.ext.six.moves import map
if sys.version_info < (2, 7):
import unittest2 as unittest
else:
import unittest
import os
import stat
import time
import tempfile
import shutil
from ioflo.aid.odicting import odict
from ioflo.aid.timing import Timer, StoreTimer
from ioflo.base import storing
from ioflo.base.consoling import getConsole
console = getConsole()
from raet import raeting, nacling
from raet.road import estating, keeping, stacking
from salt.key import RaetKey
def setUpModule():
console.reinit(verbosity=console.Wordage.concise)
def tearDownModule():
pass
class BasicTestCase(unittest.TestCase):
""""""
def setUp(self):
self.store = storing.Store(stamp=0.0)
self.timer = StoreTimer(store=self.store, duration=1.0)
self.saltDirpath = tempfile.mkdtemp(prefix="salt", suffix="main", dir='/tmp')
pkiDirpath = os.path.join(self.saltDirpath, 'pki')
if not os.path.exists(pkiDirpath):
os.makedirs(pkiDirpath)
acceptedDirpath = os.path.join(pkiDirpath, 'accepted')
if not os.path.exists(acceptedDirpath):
os.makedirs(acceptedDirpath)
pendingDirpath = os.path.join(pkiDirpath, 'pending')
if not os.path.exists(pendingDirpath):
os.makedirs(pendingDirpath)
rejectedDirpath = os.path.join(pkiDirpath, 'rejected')
if not os.path.exists(rejectedDirpath):
os.makedirs(rejectedDirpath)
self.localFilepath = os.path.join(pkiDirpath, 'local.key')
if os.path.exists(self.localFilepath):
mode = os.stat(self.localFilepath).st_mode
os.chmod(self.localFilepath, mode | stat.S_IWUSR | stat.S_IWUSR)
self.cacheDirpath = os.path.join(self.saltDirpath, 'cache')
self.sockDirpath = os.path.join(self.saltDirpath, 'sock')
self.opts = dict(
__role='master',
id='master',
pki_dir=pkiDirpath,
sock_dir=self.sockDirpath,
cachedir=self.cacheDirpath,
open_mode=False,
auto_accept=True,
transport='raet',
)
self.mainKeeper = RaetKey(opts=self.opts)
self.baseDirpath = tempfile.mkdtemp(prefix="salt", suffix="base", dir='/tmp')
def tearDown(self):
if os.path.exists(self.saltDirpath):
shutil.rmtree(self.saltDirpath)
def createRoadData(self, name, base):
'''
Creates odict and populates with data to setup road stack
{
name: stack name local estate name
dirpath: dirpath for keep files
sighex: signing key
verhex: verify key
prihex: private key
pubhex: public key
}
'''
data = odict()
data['name'] = name
data['dirpath'] = os.path.join(base, 'road', 'keep', name)
signer = nacling.Signer()
data['sighex'] = signer.keyhex
data['verhex'] = signer.verhex
privateer = nacling.Privateer()
data['prihex'] = privateer.keyhex
data['pubhex'] = privateer.pubhex
return data
def testAutoAccept(self):
'''
Basic function of RaetKey in auto accept mode
'''
console.terse("{0}\n".format(self.testAutoAccept.__doc__))
self.opts['auto_accept'] = True
self.assertTrue(self.opts['auto_accept'])
self.assertDictEqual(self.mainKeeper.all_keys(), {'accepted': [],
'local': [],
'rejected': [],
'pending': []})
localkeys = self.mainKeeper.read_local()
self.assertDictEqual(localkeys, {})
main = self.createRoadData(name='main', base=self.baseDirpath)
self.mainKeeper.write_local(main['prihex'], main['sighex'])
localkeys = self.mainKeeper.read_local()
self.assertDictEqual(localkeys,
{'priv': salt.utils.stringutils.to_str(main['prihex']),
'sign': salt.utils.stringutils.to_str(main['sighex'])})
allkeys = self.mainKeeper.all_keys()
self.assertDictEqual(allkeys, {'accepted': [],
'local': [self.localFilepath],
'rejected': [],
'pending': []})
other1 = self.createRoadData(name='other1', base=self.baseDirpath)
other2 = self.createRoadData(name='other2', base=self.baseDirpath)
status = self.mainKeeper.status(other1['name'], other1['pubhex'], other1['verhex'])
self.assertEqual(status, 'accepted')
status = self.mainKeeper.status(other2['name'], other2['pubhex'], other2['verhex'])
self.assertEqual(status, 'accepted')
allkeys = self.mainKeeper.all_keys()
self.assertDictEqual(allkeys, {'accepted': ['other1', 'other2'],
'local': [self.localFilepath],
'pending': [],
'rejected': []})
remotekeys = self.mainKeeper.read_remote(other1['name'])
self.assertDictEqual(remotekeys, {'minion_id': 'other1',
'pub': salt.utils.stringutils.to_str(other1['pubhex']),
'verify': salt.utils.stringutils.to_str(other1['verhex'])})
remotekeys = self.mainKeeper.read_remote(other2['name'])
self.assertDictEqual(remotekeys, {'minion_id': 'other2',
'pub': salt.utils.stringutils.to_str(other2['pubhex']),
'verify': salt.utils.stringutils.to_str(other2['verhex'])})
listkeys = self.mainKeeper.list_keys()
self.assertDictEqual(listkeys, {'accepted': ['other1', 'other2'],
'rejected': [],
'pending': []})
allremotekeys = self.mainKeeper.read_all_remote()
self.assertDictEqual(allremotekeys,
{'other1':
{'verify': salt.utils.stringutils.to_str(other1['verhex']),
'minion_id': 'other1',
'acceptance': 'accepted',
'pub': salt.utils.stringutils.to_str(other1['pubhex']), },
'other2':
{'verify': salt.utils.stringutils.to_str(other2['verhex']),
'minion_id': 'other2',
'acceptance': 'accepted',
'pub': salt.utils.stringutils.to_str(other2['pubhex']), }
})
def testManualAccept(self):
'''
Basic function of RaetKey in non auto accept mode
'''
console.terse("{0}\n".format(self.testAutoAccept.__doc__))
self.opts['auto_accept'] = False
self.assertFalse(self.opts['auto_accept'])
self.assertDictEqual(self.mainKeeper.all_keys(), {'accepted': [],
'local': [],
'rejected': [],
'pending': []})
localkeys = self.mainKeeper.read_local()
self.assertDictEqual(localkeys, {})
main = self.createRoadData(name='main', base=self.baseDirpath)
self.mainKeeper.write_local(main['prihex'], main['sighex'])
localkeys = self.mainKeeper.read_local()
self.assertDictEqual(localkeys,
{'priv': salt.utils.stringutils.to_str(main['prihex']),
'sign': salt.utils.stringutils.to_str(main['sighex'])})
allkeys = self.mainKeeper.all_keys()
self.assertDictEqual(allkeys, {'accepted': [],
'local': [self.localFilepath],
'rejected': [],
'pending': []})
other1 = self.createRoadData(name='other1', base=self.baseDirpath)
other2 = self.createRoadData(name='other2', base=self.baseDirpath)
status = self.mainKeeper.status(other1['name'], other1['pubhex'], other1['verhex'])
self.assertEqual(status, 'pending')
status = self.mainKeeper.status(other2['name'], other2['pubhex'], other2['verhex'])
self.assertEqual(status, 'pending')
allkeys = self.mainKeeper.all_keys()
self.assertDictEqual(allkeys, {'accepted': [],
'local': [self.localFilepath],
'pending': ['other1', 'other2'],
'rejected': []})
remotekeys = self.mainKeeper.read_remote(other1['name'])
self.assertDictEqual(remotekeys, {})
remotekeys = self.mainKeeper.read_remote(other2['name'])
self.assertDictEqual(remotekeys, {})
listkeys = self.mainKeeper.list_keys()
self.assertDictEqual(listkeys, {'accepted': [],
'rejected': [],
'pending': ['other1', 'other2']})
allremotekeys = self.mainKeeper.read_all_remote()
self.assertDictEqual(allremotekeys,
{'other1':
{'verify': salt.utils.stringutils.to_str(other1['verhex']),
'minion_id': 'other1',
'acceptance': 'pending',
'pub': salt.utils.stringutils.to_str(other1['pubhex']),
},
'other2':
{'verify': salt.utils.stringutils.to_str(other2['verhex']),
'minion_id': 'other2',
'acceptance': 'pending',
'pub': salt.utils.stringutils.to_str(other2['pubhex']),
}
})
self.mainKeeper.accept_all()
allkeys = self.mainKeeper.all_keys()
self.assertDictEqual(allkeys, {'accepted': ['other1', 'other2'],
'local': [self.localFilepath],
'pending': [],
'rejected': []})
remotekeys = self.mainKeeper.read_remote(other1['name'])
self.assertDictEqual(remotekeys, {'minion_id': 'other1',
'pub': salt.utils.stringutils.to_str(other1['pubhex']),
'verify': salt.utils.stringutils.to_str(other1['verhex'])})
remotekeys = self.mainKeeper.read_remote(other2['name'])
self.assertDictEqual(remotekeys, {'minion_id': 'other2',
'pub': salt.utils.stringutils.to_str(other2['pubhex']),
'verify': salt.utils.stringutils.to_str(other2['verhex'])})
listkeys = self.mainKeeper.list_keys()
self.assertDictEqual(listkeys, {'accepted': ['other1', 'other2'],
'rejected': [],
'pending': []})
allremotekeys = self.mainKeeper.read_all_remote()
self.assertDictEqual(allremotekeys,
{'other1':
{'verify': salt.utils.stringutils.to_str(other1['verhex']),
'minion_id': 'other1',
'acceptance': 'accepted',
'pub': salt.utils.stringutils.to_str(other1['pubhex']),
},
'other2':
{'verify': salt.utils.stringutils.to_str(other2['verhex']),
'minion_id': 'other2',
'acceptance': 'accepted',
'pub': salt.utils.stringutils.to_str(other2['pubhex']),
}
})
def testDelete(self):
'''
Basic function of RaetKey to delete key
'''
console.terse("{0}\n".format(self.testDelete.__doc__))
self.opts['auto_accept'] = True
self.assertTrue(self.opts['auto_accept'])
self.assertDictEqual(self.mainKeeper.all_keys(), {'accepted': [],
'local': [],
'rejected': [],
'pending': []})
localkeys = self.mainKeeper.read_local()
self.assertDictEqual(localkeys, {})
main = self.createRoadData(name='main', base=self.baseDirpath)
self.mainKeeper.write_local(main['prihex'], main['sighex'])
localkeys = self.mainKeeper.read_local()
self.assertDictEqual(localkeys,
{'priv': salt.utils.stringutils.to_str(main['prihex']),
'sign': salt.utils.stringutils.to_str(main['sighex'])})
allkeys = self.mainKeeper.all_keys()
self.assertDictEqual(allkeys, {'accepted': [],
'local': [self.localFilepath],
'rejected': [],
'pending': []})
other1 = self.createRoadData(name='other1', base=self.baseDirpath)
other2 = self.createRoadData(name='other2', base=self.baseDirpath)
status = self.mainKeeper.status(other1['name'], other1['pubhex'], other1['verhex'])
self.assertEqual(status, 'accepted')
status = self.mainKeeper.status(other2['name'], other2['pubhex'], other2['verhex'])
self.assertEqual(status, 'accepted')
allkeys = self.mainKeeper.all_keys()
self.assertDictEqual(allkeys, {'accepted': ['other1', 'other2'],
'local': [self.localFilepath],
'pending': [],
'rejected': []})
remotekeys = self.mainKeeper.read_remote(other1['name'])
self.assertDictEqual(remotekeys, {'minion_id': 'other1',
'pub': salt.utils.stringutils.to_str(other1['pubhex']),
'verify': salt.utils.stringutils.to_str(other1['verhex']),
})
remotekeys = self.mainKeeper.read_remote(other2['name'])
self.assertDictEqual(remotekeys, {'minion_id': 'other2',
'pub': salt.utils.stringutils.to_str(other2['pubhex']),
'verify': salt.utils.stringutils.to_str(other2['verhex']),
})
listkeys = self.mainKeeper.list_keys()
self.assertDictEqual(listkeys, {'accepted': ['other1', 'other2'],
'rejected': [],
'pending': []})
allremotekeys = self.mainKeeper.read_all_remote()
self.assertDictEqual(allremotekeys,
{'other1':
{'verify': salt.utils.stringutils.to_str(other1['verhex']),
'minion_id': 'other1',
'acceptance': 'accepted',
'pub': salt.utils.stringutils.to_str(other1['pubhex'])
},
'other2':
{'verify': salt.utils.stringutils.to_str(other2['verhex']),
'minion_id': 'other2',
'acceptance': 'accepted',
'pub': salt.utils.stringutils.to_str(other2['pubhex']),
}
})
self.mainKeeper.delete_key(match=other1['name'])
allkeys = self.mainKeeper.all_keys()
self.assertDictEqual(allkeys, {'accepted': ['other2'],
'local': [self.localFilepath],
'pending': [],
'rejected': []})
remotekeys = self.mainKeeper.read_remote(other1['name'])
self.assertDictEqual(remotekeys, {})
remotekeys = self.mainKeeper.read_remote(other2['name'])
self.assertDictEqual(remotekeys, {'minion_id': 'other2',
'pub': salt.utils.stringutils.to_str(other2['pubhex']),
'verify': salt.utils.stringutils.to_str(other2['verhex'])})
listkeys = self.mainKeeper.list_keys()
self.assertDictEqual(listkeys, {'accepted': ['other2'],
'rejected': [],
'pending': []})
allremotekeys = self.mainKeeper.read_all_remote()
self.assertDictEqual(allremotekeys,
{'other2':
{'verify': salt.utils.stringutils.to_str(other2['verhex']),
'minion_id': 'other2',
'acceptance': 'accepted',
'pub': salt.utils.stringutils.to_str(other2['pubhex']),
}
})
def runOne(test):
'''
Unittest Runner
'''
test = BasicTestCase(test)
suite = unittest.TestSuite([test])
unittest.TextTestRunner(verbosity=2).run(suite)
def runSome():
'''
Unittest runner
'''
tests = []
names = ['testAutoAccept',
'testManualAccept',
'testDelete']
tests.extend(list(list(map(BasicTestCase, names))))
suite = unittest.TestSuite(tests)
unittest.TextTestRunner(verbosity=2).run(suite)
def runAll():
'''
Unittest runner
'''
suite = unittest.TestSuite()
suite.addTest(unittest.TestLoader().loadTestsFromTestCase(BasicTestCase))
unittest.TextTestRunner(verbosity=2).run(suite)
if __name__ == '__main__' and __package__ is None:
# console.reinit(verbosity=console.Wordage.concise)
runAll() # run all unittests
# runSome() #only run some
# runOne('testDelete')

2236
salt/daemons/test/test_saltkeep.py
View file

File diff suppressed because it is too large Load diff

734
salt/daemons/test/test_stats.py
View file

@ -1,734 +0,0 @@
# -*- coding: utf-8 -*-
'''
Raet Ioflo Behavior Unittests
'''
from __future__ import absolute_import, print_function, unicode_literals
import sys
from salt.ext.six.moves import map
import importlib
# pylint: disable=blacklisted-import
if sys.version_info < (2, 7):
import unittest2 as unittest
else:
import unittest
# pylint: enable=blacklisted-import
import time
from ioflo.base.consoling import getConsole
console = getConsole()
from ioflo.aid.odicting import odict
from ioflo.test import testing
from raet.abiding import ns2u
from raet.lane.stacking import LaneStack
from raet.road.stacking import RoadStack
import salt.utils.stringutils
from salt.utils.event import tagify
def setUpModule():
console.reinit(verbosity=console.Wordage.concise)
def tearDownModule():
pass
class StatsEventerTestCase(testing.FrameIofloTestCase):
'''
Test case for Salt Raet Stats Eventer Master and Minion deeds
'''
def setUp(self):
'''
Call super if override so House Framer and Frame are setup correctly
'''
behaviors = ['salt.daemons.flo', 'salt.daemons.test.plan']
for behavior in behaviors:
mod = importlib.import_module(behavior)
super(StatsEventerTestCase, self).setUp()
def tearDown(self):
'''
Call super if override so House Framer and Frame are torn down correctly
'''
super(StatsEventerTestCase, self).tearDown()
def testMasterContextSetup(self):
'''
Test the context setup procedure used in all the consequence tests works as expected
This test intended to avoid some checks in other tests
'''
console.terse("{0}\n".format(self.testMasterContextSetup.__doc__))
act = self.addEnterDeed("TestOptsSetupMaster")
self.assertIn(act, self.frame.enacts)
self.assertEqual(act.actor, "TestOptsSetupMaster")
act = self.addEnterDeed("SaltRaetManorLaneSetup")
self.assertIn(act, self.frame.enacts)
self.assertEqual(act.actor, "SaltRaetManorLaneSetup")
act = self.addEnterDeed("SaltRaetRoadStackSetup")
self.assertIn(act, self.frame.enacts)
self.assertEqual(act.actor, "SaltRaetRoadStackSetup")
act = self.addEnterDeed("StatsMasterTestSetup")
self.assertIn(act, self.frame.enacts)
self.assertEqual(act.actor, "StatsMasterTestSetup")
act = self.addRecurDeed("SaltRaetStatsEventer")
self.assertIn(act, self.frame.reacts)
self.assertEqual(act.actor, "SaltRaetStatsEventer")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
self.assertDictEqual(
act.actor.Ioinits,
{'opts': salt.utils.stringutils.to_str('.salt.opts'),
'stats_req': salt.utils.stringutils.to_str('.salt.stats.event_req'),
'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'),
'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack')})
self.assertTrue(hasattr(act.actor, 'opts'))
self.assertTrue(hasattr(act.actor, 'stats_req'))
self.assertTrue(hasattr(act.actor, 'lane_stack'))
self.assertTrue(hasattr(act.actor, 'road_stack'))
self.assertIsInstance(act.actor.lane_stack.value, LaneStack)
self.assertIsInstance(act.actor.road_stack.value, RoadStack)
self.frame.recur()
# Close active stacks servers
act.actor.lane_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
act.actor.road_stack.value.server.close()
def testMasterRoadStats(self):
'''
Test Master Road Stats request (A1)
'''
console.terse("{0}\n".format(self.testMasterRoadStats.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("SaltRaetRoadStackSetup")
self.addEnterDeed("StatsMasterTestSetup")
act = self.addRecurDeed("SaltRaetStatsEventerMaster")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add a test stat key-value
roadStack = self.store.fetch('.salt.road.manor.stack')
laneStack = self.store.fetch('.salt.lane.manor.stack')
roadStack.value.stats['test_stats_event'] = 111
# ensure stats are equal to expected
self.assertDictEqual(roadStack.value.stats, {'test_stats_event': 111})
self.assertDictEqual(laneStack.value.stats, {})
# add stats request
testStack = self.store.fetch('.salt.test.lane.stack').value
statsReq = self.store.fetch('.salt.stats.event_req').value
tag = tagify('road', 'stats')
# road stats request
statsReq.append({'route': {'dst': (None, None, 'stats_req'),
'src': (None, testStack.local.name, None)},
'tag': tag})
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 1)
msg, sender = testStack.rxMsgs.popleft()
self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'test_stats_event': 111}})
# Close active stacks servers
act.actor.lane_stack.value.server.close()
act.actor.road_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
time.sleep(0.1)
def testMasterLaneStats(self):
'''
Test Master Road Stats request (A2)
'''
console.terse("{0}\n".format(self.testMasterLaneStats.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("SaltRaetRoadStackSetup")
self.addEnterDeed("StatsMasterTestSetup")
act = self.addRecurDeed("SaltRaetStatsEventerMaster")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add a test stat key-value
roadStack = self.store.fetch('.salt.road.manor.stack')
laneStack = self.store.fetch('.salt.lane.manor.stack')
laneStack.value.stats['test_stats_event'] = 111
# ensure stats are equal to expected
self.assertDictEqual(roadStack.value.stats, {})
self.assertDictEqual(laneStack.value.stats, {'test_stats_event': 111})
# add stats request
testStack = self.store.fetch('.salt.test.lane.stack').value
statsReq = self.store.fetch('.salt.stats.event_req').value
tag = tagify('lane', 'stats')
# lane stats request
statsReq.append({'route': {'dst': (None, None, 'stats_req'),
'src': (None, testStack.local.name, None)},
'tag': tag})
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 1)
msg, sender = testStack.rxMsgs.popleft()
self.assertDictEqual(msg, {'route': {'src': [None, 'manor', None],
'dst': [None, None, 'event_fire']},
'tag': tag,
'data': {'test_stats_event': 111}})
# Close active stacks servers
act.actor.lane_stack.value.server.close()
act.actor.road_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def testMasterStatsWrongMissingTag(self):
'''
Test Master Stats requests with unknown and missing tag (A3, A4)
'''
console.terse("{0}\n".format(self.testMasterStatsWrongMissingTag.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("SaltRaetRoadStackSetup")
self.addEnterDeed("StatsMasterTestSetup")
act = self.addRecurDeed("SaltRaetStatsEventerMaster")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add a test stat key-value
roadStack = self.store.fetch('.salt.road.manor.stack')
laneStack = self.store.fetch('.salt.lane.manor.stack')
roadStack.value.stats['test_road_stats_event'] = 111
laneStack.value.stats['test_lane_stats_event'] = 222
# ensure stats are equal to expected
self.assertDictEqual(roadStack.value.stats, {'test_road_stats_event': 111})
self.assertDictEqual(laneStack.value.stats, {'test_lane_stats_event': 222})
# add stats request
testStack = self.store.fetch('.salt.test.lane.stack').value
statsReq = self.store.fetch('.salt.stats.event_req').value
tag = 'salt/unknown/tag'
self.assertNotEqual(tag, tagify('lane', 'stats'))
self.assertNotEqual(tag, tagify('road', 'stats'))
# unknown tag in stats request
statsReq.append({'route': {'dst': (None, None, 'stats_req'),
'src': (None, testStack.local.name, None)},
'tag': tag})
# no tag in stats request
statsReq.append({'route': {'dst': (None, None, 'stats_req'),
'src': (None, testStack.local.name, None)}})
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 0)
# Close active stacks servers
act.actor.lane_stack.value.server.close()
act.actor.road_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def testMasterStatsUnknownRemote(self):
'''
Test Master Stats request with unknown remote (B1)
'''
console.terse("{0}\n".format(self.testMasterStatsUnknownRemote.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("SaltRaetRoadStackSetup")
self.addEnterDeed("StatsMasterTestSetup")
act = self.addRecurDeed("SaltRaetStatsEventerMaster")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add a test stat key-value
roadStack = self.store.fetch('.salt.road.manor.stack')
laneStack = self.store.fetch('.salt.lane.manor.stack')
roadStack.value.stats['test_road_stats_event'] = 111
laneStack.value.stats['test_lane_stats_event'] = 222
# ensure stats are equal to expected
self.assertDictEqual(roadStack.value.stats, {'test_road_stats_event': 111})
self.assertDictEqual(laneStack.value.stats, {'test_lane_stats_event': 222})
# add stats request
testStack = self.store.fetch('.salt.test.lane.stack').value
statsReq = self.store.fetch('.salt.stats.event_req').value
tag = tagify('road', 'stats')
# unknown tag in stats request
unknownName = 'unknownName'
statsReq.append({'route': {'dst': (None, None, 'stats_req'),
'src': (None, unknownName, None)},
'tag': tag})
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 0)
# Close active stacks servers
act.actor.lane_stack.value.server.close()
act.actor.road_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def testMasterStatsNoRequest(self):
'''
Test Master Stats no requests (nothing to do) (B2)
'''
console.terse("{0}\n".format(self.testMasterStatsNoRequest.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMaster")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("SaltRaetRoadStackSetup")
self.addEnterDeed("StatsMasterTestSetup")
act = self.addRecurDeed("SaltRaetStatsEventerMaster")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add a test stat key-value
roadStack = self.store.fetch('.salt.road.manor.stack')
laneStack = self.store.fetch('.salt.lane.manor.stack')
roadStack.value.stats['test_road_stats_event'] = 111
laneStack.value.stats['test_lane_stats_event'] = 222
# ensure stats are equal to expected
self.assertDictEqual(roadStack.value.stats, {'test_road_stats_event': 111})
self.assertDictEqual(laneStack.value.stats, {'test_lane_stats_event': 222})
# add stats request
testStack = self.store.fetch('.salt.test.lane.stack').value
statsReq = self.store.fetch('.salt.stats.event_req').value
# no requests
self.assertEqual(len(statsReq), 0)
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 0)
# Close active stacks servers
act.actor.lane_stack.value.server.close()
act.actor.road_stack.value.server.close()
testStack = self.store.fetch('.salt.test.lane.stack')
if testStack:
testStack.value.server.close()
def testMinionContextSetup(self):
'''
Test the context setup procedure used in all the consequence tests works as expected
This test intended to avoid some checks in other tests
'''
console.terse("{0}\n".format(self.testMinionContextSetup.__doc__))
act = self.addEnterDeed("TestOptsSetupMinion")
self.assertIn(act, self.frame.enacts)
self.assertEqual(act.actor, "TestOptsSetupMinion")
act = self.addEnterDeed("SaltRaetManorLaneSetup")
self.assertIn(act, self.frame.enacts)
self.assertEqual(act.actor, "SaltRaetManorLaneSetup")
act = self.addEnterDeed("SaltRaetRoadStackSetup")
self.assertIn(act, self.frame.enacts)
self.assertEqual(act.actor, "SaltRaetRoadStackSetup")
act = self.addEnterDeed("StatsMinionTestSetup")
self.assertIn(act, self.frame.enacts)
self.assertEqual(act.actor, "StatsMinionTestSetup")
act = self.addRecurDeed("SaltRaetStatsEventer")
self.assertIn(act, self.frame.reacts)
self.assertEqual(act.actor, "SaltRaetStatsEventer")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
self.assertDictEqual(
act.actor.Ioinits,
{'opts': salt.utils.stringutils.to_str('.salt.opts'),
'stats_req': salt.utils.stringutils.to_str('.salt.stats.event_req'),
'lane_stack': salt.utils.stringutils.to_str('.salt.lane.manor.stack'),
'road_stack': salt.utils.stringutils.to_str('.salt.road.manor.stack')})
self.assertTrue(hasattr(act.actor, 'opts'))
self.assertTrue(hasattr(act.actor, 'stats_req'))
self.assertTrue(hasattr(act.actor, 'lane_stack'))
self.assertTrue(hasattr(act.actor, 'road_stack'))
self.assertIsInstance(act.actor.lane_stack.value, LaneStack)
self.assertIsInstance(act.actor.road_stack.value, RoadStack)
self.frame.recur()
# Close active stacks servers
act.actor.lane_stack.value.server.close()
testStack = self.store.fetch('.salt.test.road.stack')
if testStack:
testStack.value.server.close()
act.actor.road_stack.value.server.close()
def testMinionRoadStats(self):
'''
Test Minion Road Stats request (A1)
'''
console.terse("{0}\n".format(self.testMinionRoadStats.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMinion")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("SaltRaetRoadStackSetup")
self.addEnterDeed("StatsMinionTestSetup")
act = self.addRecurDeed("SaltRaetStatsEventerMinion")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add a test stat key-value
roadStack = self.store.fetch('.salt.road.manor.stack')
laneStack = self.store.fetch('.salt.lane.manor.stack')
roadStack.value.stats = odict({'test_stats_event': 111})
laneStack.value.stats = odict()
# ensure stats are equal to expected
self.assertDictEqual(roadStack.value.stats, {'test_stats_event': 111})
self.assertDictEqual(laneStack.value.stats, {})
# add stats request
testStack = self.store.fetch('.salt.test.road.stack').value
statsReq = self.store.fetch('.salt.stats.event_req').value
tag = tagify('road', 'stats')
minionName = roadStack.value.local.name
masterName = testStack.local.name
# road stats request
statsReq.append({'route': {'dst': (minionName, None, 'stats_req'),
'src': (masterName, None, None)},
'tag': tag})
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 1)
msg, sender = testStack.rxMsgs.popleft()
self.assertDictEqual(msg, {'route': {'src': [ns2u(minionName), 'manor', None],
'dst': [ns2u(masterName), None, 'event_fire']},
'tag': ns2u(tag),
'data': {'test_stats_event': 111}})
# Close active stacks servers
act.actor.lane_stack.value.server.close()
act.actor.road_stack.value.server.close()
testStack = self.store.fetch('.salt.test.road.stack')
if testStack:
testStack.value.server.close()
def testMinionLaneStats(self):
'''
Test Minion Road Stats request (A2)
'''
console.terse("{0}\n".format(self.testMinionLaneStats.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMinion")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("SaltRaetRoadStackSetup")
self.addEnterDeed("StatsMinionTestSetup")
act = self.addRecurDeed("SaltRaetStatsEventerMinion")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add a test stat key-value
roadStack = self.store.fetch('.salt.road.manor.stack')
laneStack = self.store.fetch('.salt.lane.manor.stack')
roadStack.value.stats = odict()
laneStack.value.stats = odict({'test_stats_event': 111})
# ensure stats are equal to expected
self.assertDictEqual(roadStack.value.stats, {})
self.assertDictEqual(laneStack.value.stats, {'test_stats_event': 111})
# add stats request
testStack = self.store.fetch('.salt.test.road.stack').value
statsReq = self.store.fetch('.salt.stats.event_req').value
tag = tagify('lane', 'stats')
minionName = roadStack.value.local.name
masterName = testStack.local.name
# lane stats request
statsReq.append({'route': {'dst': (minionName, None, 'stats_req'),
'src': (masterName, None, None)},
'tag': tag})
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 1)
msg, sender = testStack.rxMsgs.popleft()
self.assertDictEqual(msg, {'route': {'src': [ns2u(minionName), 'manor', None],
'dst': [ns2u(masterName), None, 'event_fire']},
'tag': ns2u(tag),
'data': {'test_stats_event': 111}})
# Close active stacks servers
act.actor.lane_stack.value.server.close()
act.actor.road_stack.value.server.close()
testStack = self.store.fetch('.salt.test.road.stack')
if testStack:
testStack.value.server.close()
def testMinionStatsWrongMissingTag(self):
'''
Test Minion Stats requests with unknown and missing tag (A3, A4)
'''
console.terse("{0}\n".format(self.testMinionStatsWrongMissingTag.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMinion")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("SaltRaetRoadStackSetup")
self.addEnterDeed("StatsMinionTestSetup")
act = self.addRecurDeed("SaltRaetStatsEventerMinion")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add a test stat key-value
roadStack = self.store.fetch('.salt.road.manor.stack')
laneStack = self.store.fetch('.salt.lane.manor.stack')
roadStack.value.stats = odict({'test_road_stats_event': 111})
laneStack.value.stats = odict({'test_lane_stats_event': 222})
# ensure stats are equal to expected
self.assertDictEqual(roadStack.value.stats, {'test_road_stats_event': 111})
self.assertDictEqual(laneStack.value.stats, {'test_lane_stats_event': 222})
# add stats request
testStack = self.store.fetch('.salt.test.road.stack').value
statsReq = self.store.fetch('.salt.stats.event_req').value
tag = 'salt/unknown/tag'
self.assertNotEqual(tag, tagify('lane', 'stats'))
self.assertNotEqual(tag, tagify('road', 'stats'))
minionName = roadStack.value.local.name
masterName = testStack.local.name
# unknown tag in stats request
statsReq.append({'route': {'dst': (minionName, None, 'stats_req'),
'src': (masterName, None, None)},
'tag': tag})
# no tag in stats request
statsReq.append({'route': {'dst': (minionName, None, 'stats_req'),
'src': (masterName, None, None)}})
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 0)
# Close active stacks servers
act.actor.lane_stack.value.server.close()
act.actor.road_stack.value.server.close()
testStack = self.store.fetch('.salt.test.road.stack')
if testStack:
testStack.value.server.close()
def testMinionStatsUnknownRemote(self):
'''
Test Minion Stats request with unknown remote (B1)
'''
console.terse("{0}\n".format(self.testMinionStatsUnknownRemote.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMinion")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("SaltRaetRoadStackSetup")
self.addEnterDeed("StatsMinionTestSetup")
act = self.addRecurDeed("SaltRaetStatsEventerMinion")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add a test stat key-value
roadStack = self.store.fetch('.salt.road.manor.stack')
laneStack = self.store.fetch('.salt.lane.manor.stack')
roadStack.value.stats = odict({'test_road_stats_event': 111})
laneStack.value.stats = odict({'test_lane_stats_event': 222})
# ensure stats are equal to expected
self.assertDictEqual(roadStack.value.stats, {'test_road_stats_event': 111})
self.assertDictEqual(laneStack.value.stats, {'test_lane_stats_event': 222})
# add stats request
testStack = self.store.fetch('.salt.test.road.stack').value
statsReq = self.store.fetch('.salt.stats.event_req').value
tag = tagify('road', 'stats')
minionName = roadStack.value.local.name
# unknown remote (src) name in stats request
unknownName = 'unknownName'
statsReq.append({'route': {'dst': (minionName, None, 'stats_req'),
'src': (unknownName, None, None)},
'tag': tag})
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 0)
# Close active stacks servers
act.actor.lane_stack.value.server.close()
act.actor.road_stack.value.server.close()
testStack = self.store.fetch('.salt.test.road.stack')
if testStack:
testStack.value.server.close()
def testMinionStatsNoRequest(self):
'''
Test Minion Stats no requests (nothing to do) (B2)
'''
console.terse("{0}\n".format(self.testMinionStatsNoRequest.__doc__))
# Bootstrap
self.addEnterDeed("TestOptsSetupMinion")
self.addEnterDeed("SaltRaetManorLaneSetup")
self.addEnterDeed("SaltRaetRoadStackSetup")
self.addEnterDeed("StatsMinionTestSetup")
act = self.addRecurDeed("SaltRaetStatsEventerMinion")
self.resolve() # resolve House, Framer, Frame, Acts, Actors
self.frame.enter()
# Prepare
# add a test stat key-value
roadStack = self.store.fetch('.salt.road.manor.stack')
laneStack = self.store.fetch('.salt.lane.manor.stack')
roadStack.value.stats = odict({'test_road_stats_event': 111})
laneStack.value.stats = odict({'test_lane_stats_event': 222})
# ensure stats are equal to expected
self.assertDictEqual(roadStack.value.stats, {'test_road_stats_event': 111})
self.assertDictEqual(laneStack.value.stats, {'test_lane_stats_event': 222})
# clear lane stack remotes
# add stats request
testStack = self.store.fetch('.salt.test.road.stack').value
statsReq = self.store.fetch('.salt.stats.event_req').value
# no request
self.assertEqual(len(statsReq), 0)
# Test
self.frame.recur() # run in frame
# Check
self.assertEqual(len(testStack.rxMsgs), 0)
testStack.serviceAll()
self.assertEqual(len(testStack.rxMsgs), 0)
# Close active stacks servers
act.actor.lane_stack.value.server.close()
act.actor.road_stack.value.server.close()
testStack = self.store.fetch('.salt.test.road.stack')
if testStack:
testStack.value.server.close()
def runOne(test):
'''
Unittest Runner
'''
test = StatsEventerTestCase(test)
suite = unittest.TestSuite([test])
unittest.TextTestRunner(verbosity=2).run(suite)
def runSome():
'''
Unittest runner
'''
tests = []
names = [
'testMasterContextSetup',
'testMasterRoadStats',
'testMasterLaneStats',
'testMasterStatsWrongMissingTag',
'testMasterStatsUnknownRemote',
'testMasterStatsNoRequest',
'testMinionContextSetup',
'testMinionRoadStats',
'testMinionLaneStats',
'testMinionStatsWrongMissingTag',
'testMinionStatsUnknownRemote',
'testMinionStatsNoRequest',
]
tests.extend(list(map(StatsEventerTestCase, names)))
suite = unittest.TestSuite(tests)
unittest.TextTestRunner(verbosity=2).run(suite)
def runAll():
'''
Unittest runner
'''
suite = unittest.TestSuite()
suite.addTest(unittest.TestLoader().loadTestsFromTestCase(StatsEventerTestCase))
unittest.TextTestRunner(verbosity=2).run(suite)
if __name__ == '__main__' and __package__ is None:
# console.reinit(verbosity=console.Wordage.concise)
runAll() # run all unittests
#runSome() # only run some
#runOne('testMasterLaneStats')

600
salt/key.py
View file

@ -7,11 +7,8 @@ used to manage salt keys directly without interfacing with the CLI.
# Import python libs # Import python libs
from __future__ import absolute_import, print_function, unicode_literals from __future__ import absolute_import, print_function, unicode_literals
import os import os
import copy
import stat
import shutil import shutil
import fnmatch import fnmatch
import hashlib
import logging import logging
# Import salt libs # Import salt libs
@ -38,20 +35,11 @@ from salt.ext import six
from salt.ext.six.moves import input, zip_longest from salt.ext.six.moves import input, zip_longest
# pylint: enable=import-error,no-name-in-module,redefined-builtin # pylint: enable=import-error,no-name-in-module,redefined-builtin
# Import third party libs
try:
import msgpack
except ImportError:
pass
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
def get_key(opts): def get_key(opts):
if opts['transport'] in ('zeromq', 'tcp'): return Key(opts)
return Key(opts)
else:
return RaetKey(opts)
class KeyCLI(object): class KeyCLI(object):
@ -67,10 +55,7 @@ class KeyCLI(object):
def __init__(self, opts): def __init__(self, opts):
self.opts = opts self.opts = opts
self.client = salt.wheel.WheelClient(opts) self.client = salt.wheel.WheelClient(opts)
if self.opts['transport'] in ('zeromq', 'tcp'): self.key = Key
self.key = Key
else:
self.key = RaetKey
# instantiate the key object for masterless mode # instantiate the key object for masterless mode
if not opts.get('eauth'): if not opts.get('eauth'):
self.key = self.key(opts) self.key = self.key(opts)
@ -299,71 +284,6 @@ class KeyCLI(object):
return ret return ret
class MultiKeyCLI(KeyCLI):
'''
Manage multiple key backends from the CLI
'''
def __init__(self, opts):
opts['__multi_key'] = True
super(MultiKeyCLI, self).__init__(opts)
# Remove the key attribute set in KeyCLI.__init__
delattr(self, 'key')
zopts = copy.copy(opts)
ropts = copy.copy(opts)
self.keys = {}
zopts['transport'] = 'zeromq'
self.keys['ZMQ Keys'] = KeyCLI(zopts)
ropts['transport'] = 'raet'
self.keys['RAET Keys'] = KeyCLI(ropts)
def _call_all(self, fun, *args):
'''
Call the given function on all backend keys
'''
for kback in self.keys:
print(kback)
getattr(self.keys[kback], fun)(*args)
def list_status(self, status):
self._call_all('list_status', status)
def list_all(self):
self._call_all('list_all')
def accept(self, match, include_rejected=False, include_denied=False):
self._call_all('accept', match, include_rejected, include_denied)
def accept_all(self, include_rejected=False, include_denied=False):
self._call_all('accept_all', include_rejected, include_denied)
def delete(self, match):
self._call_all('delete', match)
def delete_all(self):
self._call_all('delete_all')
def reject(self, match, include_accepted=False, include_denied=False):
self._call_all('reject', match, include_accepted, include_denied)
def reject_all(self, include_accepted=False, include_denied=False):
self._call_all('reject_all', include_accepted, include_denied)
def print_key(self, match):
self._call_all('print_key', match)
def print_all(self):
self._call_all('print_all')
def finger(self, match, hash_type):
self._call_all('finger', match, hash_type)
def finger_all(self, hash_type):
self._call_all('finger_all', hash_type)
def prep_signature(self):
self._call_all('prep_signature')
class Key(object): class Key(object):
''' '''
The object that encapsulates saltkey actions The object that encapsulates saltkey actions
@ -599,12 +519,6 @@ class Key(object):
''' '''
Return a dict of managed keys and what the key status are Return a dict of managed keys and what the key status are
''' '''
key_dirs = []
# We have to differentiate between RaetKey._check_minions_directories
# and Zeromq-Keys. Raet-Keys only have three states while ZeroMQ-keys
# havd an additional 'denied' state.
key_dirs = self._check_minions_directories() key_dirs = self._check_minions_directories()
ret = {} ret = {}
@ -962,513 +876,3 @@ class Key(object):
path = os.path.join(self.opts['pki_dir'], status, key) path = os.path.join(self.opts['pki_dir'], status, key)
ret[status][key] = salt.utils.crypt.pem_finger(path, sum_type=hash_type) ret[status][key] = salt.utils.crypt.pem_finger(path, sum_type=hash_type)
return ret return ret
class RaetKey(Key):
'''
Manage keys from the raet backend
'''
ACC = 'accepted'
PEND = 'pending'
REJ = 'rejected'
DEN = None
def __init__(self, opts):
Key.__init__(self, opts)
self.auto_key = salt.daemons.masterapi.AutoKey(self.opts)
self.serial = salt.payload.Serial(self.opts)
def _check_minions_directories(self):
'''
Return the minion keys directory paths
'''
accepted = os.path.join(self.opts['pki_dir'], self.ACC)
pre = os.path.join(self.opts['pki_dir'], self.PEND)
rejected = os.path.join(self.opts['pki_dir'], self.REJ)
return accepted, pre, rejected, None
def check_minion_cache(self, preserve_minions=False):
'''
Check the minion cache to make sure that old minion data is cleared
'''
keys = self.list_keys()
minions = []
for key, val in six.iteritems(keys):
minions.extend(val)
m_cache = os.path.join(self.opts['cachedir'], 'minions')
if not self.opts.get('preserve_minion_cache', False):
if os.path.isdir(m_cache):
for minion in os.listdir(m_cache):
if minion not in minions and minion not in preserve_minions:
try:
shutil.rmtree(os.path.join(m_cache, minion))
except (OSError, IOError) as ex:
log.warning('RaetKey: Delete cache for %s got OSError/IOError: %s \n',
minion,
ex)
continue
cache = salt.cache.factory(self.opts)
clist = cache.list(self.ACC)
if clist:
for minion in clist:
if minion not in minions and minion not in preserve_minions:
cache.flush('{0}/{1}'.format(self.ACC, minion))
kind = self.opts.get('__role', '') # application kind
if kind not in salt.utils.kinds.APPL_KINDS:
emsg = ("Invalid application kind = '{0}'.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
role = self.opts.get('id', '')
if not role:
emsg = ("Invalid id.")
log.error(emsg + "\n")
raise ValueError(emsg)
name = "{0}_{1}".format(role, kind)
road_cache = os.path.join(self.opts['cachedir'],
'raet',
name,
'remote')
if os.path.isdir(road_cache):
for road in os.listdir(road_cache):
root, ext = os.path.splitext(road)
if ext not in ('.json', '.msgpack'):
continue
prefix, sep, name = root.partition('.')
if not name or prefix != 'estate':
continue
path = os.path.join(road_cache, road)
with salt.utils.files.fopen(path, 'rb') as fp_:
if ext == '.json':
data = salt.utils.json.load(fp_)
elif ext == '.msgpack':
data = msgpack.load(fp_)
role = salt.utils.stringutils.to_unicode(data['role'])
if role not in minions:
os.remove(path)
def gen_keys(self, keydir=None, keyname=None, keysize=None, user=None):
'''
Use libnacl to generate and safely save a private key
'''
import libnacl.dual # pylint: disable=import-error,3rd-party-module-not-gated
d_key = libnacl.dual.DualSecret()
keydir, keyname, _, _ = self._get_key_attrs(keydir, keyname,
keysize, user)
path = '{0}.key'.format(os.path.join(
keydir,
keyname))
d_key.save(path, 'msgpack')
def check_master(self):
'''
Log if the master is not running
NOT YET IMPLEMENTED
'''
return True
def local_keys(self):
'''
Return a dict of local keys
'''
ret = {'local': []}
fn_ = os.path.join(self.opts['pki_dir'], 'local.key')
if os.path.isfile(fn_):
ret['local'].append(fn_)
return ret
def status(self, minion_id, pub, verify):
'''
Accepts the minion id, device id, curve public and verify keys.
If the key is not present, put it in pending and return "pending",
If the key has been accepted return "accepted"
if the key should be rejected, return "rejected"
'''
acc, pre, rej, _ = self._check_minions_directories() # pylint: disable=W0632
acc_path = os.path.join(acc, minion_id)
pre_path = os.path.join(pre, minion_id)
rej_path = os.path.join(rej, minion_id)
# open mode is turned on, force accept the key
pub = salt.utils.stringutils.to_str(pub)
verify = salt.utils.stringutils.to_str(verify)
keydata = {
'minion_id': minion_id,
'pub': pub,
'verify': verify}
if self.opts['open_mode']: # always accept and overwrite
with salt.utils.files.fopen(acc_path, 'w+b') as fp_:
fp_.write(self.serial.dumps(keydata))
return self.ACC
if os.path.isfile(rej_path):
log.debug("Rejection Reason: Keys already rejected.\n")
return self.REJ
elif os.path.isfile(acc_path):
# The minion id has been accepted, verify the key strings
with salt.utils.files.fopen(acc_path, 'rb') as fp_:
keydata = self.serial.loads(fp_.read())
if keydata['pub'] == pub and keydata['verify'] == verify:
return self.ACC
else:
log.debug("Rejection Reason: Keys not match prior accepted.\n")
return self.REJ
elif os.path.isfile(pre_path):
auto_reject = self.auto_key.check_autoreject(minion_id)
auto_sign = self.auto_key.check_autosign(minion_id)
with salt.utils.files.fopen(pre_path, 'rb') as fp_:
keydata = self.serial.loads(fp_.read())
if keydata['pub'] == pub and keydata['verify'] == verify:
if auto_reject:
self.reject(minion_id)
log.debug("Rejection Reason: Auto reject pended.\n")
return self.REJ
elif auto_sign:
self.accept(minion_id)
return self.ACC
return self.PEND
else:
log.debug("Rejection Reason: Keys not match prior pended.\n")
return self.REJ
# This is a new key, evaluate auto accept/reject files and place
# accordingly
auto_reject = self.auto_key.check_autoreject(minion_id)
auto_sign = self.auto_key.check_autosign(minion_id)
if self.opts['auto_accept']:
w_path = acc_path
ret = self.ACC
elif auto_sign:
w_path = acc_path
ret = self.ACC
elif auto_reject:
w_path = rej_path
log.debug("Rejection Reason: Auto reject new.\n")
ret = self.REJ
else:
w_path = pre_path
ret = self.PEND
with salt.utils.files.fopen(w_path, 'w+b') as fp_:
fp_.write(self.serial.dumps(keydata))
return ret
def _get_key_str(self, minion_id, status):
'''
Return the key string in the form of:
pub: <pub>
verify: <verify>
'''
path = os.path.join(self.opts['pki_dir'], status, minion_id)
with salt.utils.files.fopen(path, 'rb') as fp_:
keydata = self.serial.loads(fp_.read())
return 'pub: {0}\nverify: {1}'.format(
keydata['pub'],
keydata['verify'])
def _get_key_finger(self, path):
'''
Return a sha256 kingerprint for the key
'''
with salt.utils.files.fopen(path, 'rb') as fp_:
keydata = self.serial.loads(fp_.read())
key = 'pub: {0}\nverify: {1}'.format(
keydata['pub'],
keydata['verify'])
return hashlib.sha256(key).hexdigest()
def key_str(self, match):
'''
Return the specified public key or keys based on a glob
'''
ret = {}
for status, keys in six.iteritems(self.name_match(match)):
ret[status] = {}
for key in salt.utils.data.sorted_ignorecase(keys):
ret[status][key] = self._get_key_str(key, status)
return ret
def key_str_all(self):
'''
Return all managed key strings
'''
ret = {}
for status, keys in six.iteritems(self.list_keys()):
ret[status] = {}
for key in salt.utils.data.sorted_ignorecase(keys):
ret[status][key] = self._get_key_str(key, status)
return ret
def accept(self, match=None, match_dict=None, include_rejected=False, include_denied=False):
'''
Accept public keys. If "match" is passed, it is evaluated as a glob.
Pre-gathered matches can also be passed via "match_dict".
'''
if match is not None:
matches = self.name_match(match)
elif match_dict is not None and isinstance(match_dict, dict):
matches = match_dict
else:
matches = {}
keydirs = [self.PEND]
if include_rejected:
keydirs.append(self.REJ)
if include_denied:
keydirs.append(self.DEN)
for keydir in keydirs:
for key in matches.get(keydir, []):
try:
shutil.move(
os.path.join(
self.opts['pki_dir'],
keydir,
key),
os.path.join(
self.opts['pki_dir'],
self.ACC,
key)
)
except (IOError, OSError):
pass
return (
self.name_match(match) if match is not None
else self.dict_match(matches)
)
def accept_all(self):
'''
Accept all keys in pre
'''
keys = self.list_keys()
for key in keys[self.PEND]:
try:
shutil.move(
os.path.join(
self.opts['pki_dir'],
self.PEND,
key),
os.path.join(
self.opts['pki_dir'],
self.ACC,
key)
)
except (IOError, OSError):
pass
return self.list_keys()
def delete_key(self,
match=None,
match_dict=None,
preserve_minions=None,
revoke_auth=False):
'''
Delete public keys. If "match" is passed, it is evaluated as a glob.
Pre-gathered matches can also be passed via "match_dict".
'''
if match is not None:
matches = self.name_match(match)
elif match_dict is not None and isinstance(match_dict, dict):
matches = match_dict
else:
matches = {}
for status, keys in six.iteritems(matches):
for key in keys:
if revoke_auth:
if self.opts.get('rotate_aes_key') is False:
print('Immediate auth revocation specified but AES key rotation not allowed. '
'Minion will not be disconnected until the master AES key is rotated.')
else:
try:
client = salt.client.get_local_client(mopts=self.opts)
client.cmd_async(key, 'saltutil.revoke_auth')
except salt.exceptions.SaltClientError:
print('Cannot contact Salt master. '
'Connection for {0} will remain up until '
'master AES key is rotated or auth is revoked '
'with \'saltutil.revoke_auth\'.'.format(key))
try:
os.remove(os.path.join(self.opts['pki_dir'], status, key))
except (OSError, IOError):
pass
if self.opts.get('preserve_minions') is True:
self.check_minion_cache(preserve_minions=matches.get('minions', []))
else:
self.check_minion_cache()
return (
self.name_match(match) if match is not None
else self.dict_match(matches)
)
def delete_all(self):
'''
Delete all keys
'''
for status, keys in six.iteritems(self.list_keys()):
for key in keys:
try:
os.remove(os.path.join(self.opts['pki_dir'], status, key))
except (OSError, IOError):
pass
self.check_minion_cache()
return self.list_keys()
def reject(self, match=None, match_dict=None, include_accepted=False, include_denied=False):
'''
Reject public keys. If "match" is passed, it is evaluated as a glob.
Pre-gathered matches can also be passed via "match_dict".
'''
if match is not None:
matches = self.name_match(match)
elif match_dict is not None and isinstance(match_dict, dict):
matches = match_dict
else:
matches = {}
keydirs = [self.PEND]
if include_accepted:
keydirs.append(self.ACC)
if include_denied:
keydirs.append(self.DEN)
for keydir in keydirs:
for key in matches.get(keydir, []):
try:
shutil.move(
os.path.join(
self.opts['pki_dir'],
keydir,
key),
os.path.join(
self.opts['pki_dir'],
self.REJ,
key)
)
except (IOError, OSError):
pass
self.check_minion_cache()
return (
self.name_match(match) if match is not None
else self.dict_match(matches)
)
def reject_all(self):
'''
Reject all keys in pre
'''
keys = self.list_keys()
for key in keys[self.PEND]:
try:
shutil.move(
os.path.join(
self.opts['pki_dir'],
self.PEND,
key),
os.path.join(
self.opts['pki_dir'],
self.REJ,
key)
)
except (IOError, OSError):
pass
self.check_minion_cache()
return self.list_keys()
def finger(self, match, hash_type=None):
'''
Return the fingerprint for a specified key
'''
if hash_type is None:
hash_type = __opts__['hash_type']
matches = self.name_match(match, True)
ret = {}
for status, keys in six.iteritems(matches):
ret[status] = {}
for key in keys:
if status == 'local':
path = os.path.join(self.opts['pki_dir'], key)
else:
path = os.path.join(self.opts['pki_dir'], status, key)
ret[status][key] = self._get_key_finger(path)
return ret
def finger_all(self, hash_type=None):
'''
Return fingerprints for all keys
'''
if hash_type is None:
hash_type = __opts__['hash_type']
ret = {}
for status, keys in six.iteritems(self.list_keys()):
ret[status] = {}
for key in keys:
if status == 'local':
path = os.path.join(self.opts['pki_dir'], key)
else:
path = os.path.join(self.opts['pki_dir'], status, key)
ret[status][key] = self._get_key_finger(path)
return ret
def read_all_remote(self):
'''
Return a dict of all remote key data
'''
data = {}
for status, mids in six.iteritems(self.list_keys()):
for mid in mids:
keydata = self.read_remote(mid, status)
if keydata:
keydata['acceptance'] = status
data[mid] = keydata
return data
def read_remote(self, minion_id, status=ACC):
'''
Read in a remote key of status
'''
path = os.path.join(self.opts['pki_dir'], status, minion_id)
if not os.path.isfile(path):
return {}
with salt.utils.files.fopen(path, 'rb') as fp_:
return self.serial.loads(fp_.read())
def read_local(self):
'''
Read in the local private keys, return an empy dict if the keys do not
exist
'''
path = os.path.join(self.opts['pki_dir'], 'local.key')
if not os.path.isfile(path):
return {}
with salt.utils.files.fopen(path, 'rb') as fp_:
return self.serial.loads(fp_.read())
def write_local(self, priv, sign):
'''
Write the private key and the signing key to a file on disk
'''
keydata = {'priv': priv,
'sign': sign}
path = os.path.join(self.opts['pki_dir'], 'local.key')
with salt.utils.files.set_umask(0o277):
if os.path.exists(path):
#mode = os.stat(path).st_mode
os.chmod(path, stat.S_IWUSR | stat.S_IRUSR)
with salt.utils.files.fopen(path, 'w+') as fp_:
fp_.write(self.serial.dumps(keydata))
os.chmod(path, stat.S_IRUSR)
def delete_local(self):
'''
Delete the local private key file
'''
path = os.path.join(self.opts['pki_dir'], 'local.key')
if os.path.isfile(path):
os.remove(path)
def delete_pki_dir(self):
'''
Delete the private key directory
'''
path = self.opts['pki_dir']
if os.path.exists(path):
shutil.rmtree(path)

56
salt/master.py
View file

@ -13,7 +13,6 @@ import os
import re import re
import sys import sys
import time import time
import errno
import signal import signal
import stat import stat
import logging import logging
@ -2359,58 +2358,3 @@ class ClearFuncs(object):
Send the load back to the sender. Send the load back to the sender.
''' '''
return clear_load return clear_load
class FloMWorker(MWorker):
'''
Change the run and bind to be ioflo friendly
'''
def __init__(self,
opts,
key,
):
MWorker.__init__(self, opts, key)
def setup(self):
'''
Prepare the needed objects and socket for iteration within ioflo
'''
salt.utils.crypt.appendproctitle(self.__class__.__name__)
self.clear_funcs = salt.master.ClearFuncs(
self.opts,
self.key,
)
self.aes_funcs = salt.master.AESFuncs(self.opts)
self.context = zmq.Context(1)
self.socket = self.context.socket(zmq.REP)
if self.opts.get('ipc_mode', '') == 'tcp':
self.w_uri = 'tcp://127.0.0.1:{0}'.format(
self.opts.get('tcp_master_workers', 4515)
)
else:
self.w_uri = 'ipc://{0}'.format(
os.path.join(self.opts['sock_dir'], 'workers.ipc')
)
log.info('ZMQ Worker binding to socket %s', self.w_uri)
self.poller = zmq.Poller()
self.poller.register(self.socket, zmq.POLLIN)
self.socket.connect(self.w_uri)
def handle_request(self):
'''
Handle a single request
'''
try:
polled = self.poller.poll(1)
if polled:
package = self.socket.recv()
self._update_aes()
payload = self.serial.loads(package)
ret = self.serial.dumps(self._handle_payload(payload))
self.socket.send(ret)
except KeyboardInterrupt:
raise
except Exception as exc:
# Properly handle EINTR from SIGUSR1
if isinstance(exc, zmq.ZMQError) and exc.errno == errno.EINTR:
return

13
salt/modules/event.py
View file

@ -45,19 +45,6 @@ def fire_master(data, tag, preload=None):
# We can't send an event if we're in masterless mode # We can't send an event if we're in masterless mode
log.warning('Local mode detected. Event with tag %s will NOT be sent.', tag) log.warning('Local mode detected. Event with tag %s will NOT be sent.', tag)
return False return False
if __opts__['transport'] == 'raet':
channel = salt.transport.client.ReqChannel.factory(__opts__)
load = {'id': __opts__['id'],
'tag': tag,
'data': data,
'cmd': '_minion_event'}
try:
channel.send(load)
except Exception:
pass
finally:
channel.close()
return True
if preload or __opts__.get('__cli') == 'salt-call': if preload or __opts__.get('__cli') == 'salt-call':
# If preload is specified, we must send a raw event (this is # If preload is specified, we must send a raw event (this is

23
salt/modules/minion.py
View file

@ -35,20 +35,12 @@ def list_():
salt 'master' minion.list salt 'master' minion.list
''' '''
pki_dir = __salt__['config.get']('pki_dir', '') pki_dir = __salt__['config.get']('pki_dir', '')
transport = __salt__['config.get']('transport', '')
# We have to replace the minion/master directories # We have to replace the minion/master directories
pki_dir = pki_dir.replace('minion', 'master') pki_dir = pki_dir.replace('minion', 'master')
# The source code below is (nearly) a copy of salt.key.Key.list_keys # The source code below is (nearly) a copy of salt.key.Key.list_keys
key_dirs = _check_minions_directories(pki_dir)
# We have to differentiate between RaetKey._check_minions_directories
# and Zeromq-Keys. Raet-Keys only have three states while ZeroMQ-keys
# have an additional 'denied' state.
if transport in ('zeromq', 'tcp'):
key_dirs = _check_minions_directories(pki_dir)
else:
key_dirs = _check_minions_directories_raetkey(pki_dir)
ret = {} ret = {}
@ -80,19 +72,6 @@ def _check_minions_directories(pki_dir):
return minions_accepted, minions_pre, minions_rejected, minions_denied return minions_accepted, minions_pre, minions_rejected, minions_denied
def _check_minions_directories_raetkey(pki_dir):
'''
Return the minion keys directory paths.
This function is a copy of salt.key.RaetKey._check_minions_directories.
'''
accepted = os.path.join(pki_dir, salt.key.RaetKey.ACC)
pre = os.path.join(pki_dir, salt.key.RaetKey.PEND)
rejected = os.path.join(pki_dir, salt.key.RaetKey.REJ)
return accepted, pre, rejected
def kill(timeout=15): def kill(timeout=15):
''' '''
Kill the salt minion. Kill the salt minion.

241
salt/modules/raet_publish.py
View file

@ -1,241 +0,0 @@
# -*- coding: utf-8 -*-
'''
Publish a command from a minion to a target
'''
from __future__ import absolute_import, unicode_literals, print_function
# Import python libs
import time
import logging
# Import salt libs
import salt.payload
import salt.utils.args
import salt.transport.client
from salt.exceptions import SaltReqTimeoutError
# Import 3rd party libs
from salt.ext import six
log = logging.getLogger(__name__)
__virtualname__ = 'publish'
def __virtual__():
return __virtualname__ if __opts__.get('transport', '') == 'raet' else False
def _parse_args(arg):
'''
yamlify `arg` and ensure it's outermost datatype is a list
'''
yaml_args = salt.utils.args.yamlify_arg(arg)
if yaml_args is None:
return []
elif not isinstance(yaml_args, list):
return [yaml_args]
else:
return yaml_args
def _publish(
tgt,
fun,
arg=None,
tgt_type='glob',
returner='',
timeout=5,
form='clean'):
'''
Publish a command from the minion out to other minions, publications need
to be enabled on the Salt master and the minion needs to have permission
to publish the command. The Salt master will also prevent a recursive
publication loop, this means that a minion cannot command another minion
to command another minion as that would create an infinite command loop.
The arguments sent to the minion publish function are separated with
commas. This means that for a minion executing a command with multiple
args it will look like this::
salt system.example.com publish.publish '*' user.add 'foo,1020,1020'
CLI Example:
.. code-block:: bash
salt system.example.com publish.publish '*' cmd.run 'ls -la /tmp'
'''
if fun == 'publish.publish':
log.info('Function name is \'publish.publish\'. Returning {}')
return {}
arg = _parse_args(arg)
load = {'cmd': 'minion_pub',
'fun': fun,
'arg': arg,
'tgt': tgt,
'tgt_type': tgt_type,
'ret': returner,
'tmo': timeout,
'form': form,
'id': __opts__['id']}
channel = salt.transport.client.ReqChannel.factory(__opts__)
try:
try:
peer_data = channel.send(load)
except SaltReqTimeoutError:
return '\'{0}\' publish timed out'.format(fun)
if not peer_data:
return {}
# CLI args are passed as strings, re-cast to keep time.sleep happy
time.sleep(float(timeout))
load = {'cmd': 'pub_ret',
'id': __opts__['id'],
'jid': six.text_type(peer_data['jid'])}
ret = channel.send(load)
if form == 'clean':
cret = {}
for host in ret:
cret[host] = ret[host]['ret']
return cret
else:
return ret
finally:
channel.close()
def publish(tgt,
fun,
arg=None,
tgt_type='glob',
returner='',
timeout=5):
'''
Publish a command from the minion out to other minions.
Publications need to be enabled on the Salt master and the minion
needs to have permission to publish the command. The Salt master
will also prevent a recursive publication loop, this means that a
minion cannot command another minion to command another minion as
that would create an infinite command loop.
The ``tgt_type`` argument is used to pass a target other than a glob into
the execution, the available options are:
- glob
- pcre
- grain
- grain_pcre
- pillar
- pillar_pcre
- ipcidr
- range
- compound
.. versionchanged:: 2017.7.0
The ``expr_form`` argument has been renamed to ``tgt_type``, earlier
releases must use ``expr_form``.
The arguments sent to the minion publish function are separated with
commas. This means that for a minion executing a command with multiple
args it will look like this:
.. code-block:: bash
salt system.example.com publish.publish '*' user.add 'foo,1020,1020'
salt system.example.com publish.publish 'os:Fedora' network.interfaces '' grain
CLI Example:
.. code-block:: bash
salt system.example.com publish.publish '*' cmd.run 'ls -la /tmp'
.. admonition:: Attention
If you need to pass a value to a function argument and that value
contains an equal sign, you **must** include the argument name.
For example:
.. code-block:: bash
salt '*' publish.publish test.kwarg arg='cheese=spam'
'''
return _publish(tgt,
fun,
arg=arg,
tgt_type=tgt_type,
returner=returner,
timeout=timeout,
form='clean')
def full_data(tgt,
fun,
arg=None,
tgt_type='glob',
returner='',
timeout=5):
'''
Return the full data about the publication, this is invoked in the same
way as the publish function
CLI Example:
.. code-block:: bash
salt system.example.com publish.full_data '*' cmd.run 'ls -la /tmp'
.. admonition:: Attention
If you need to pass a value to a function argument and that value
contains an equal sign, you **must** include the argument name.
For example:
.. code-block:: bash
salt '*' publish.full_data test.kwarg arg='cheese=spam'
'''
return _publish(tgt,
fun,
arg=arg,
tgt_type=tgt_type,
returner=returner,
timeout=timeout,
form='full')
def runner(fun, arg=None, timeout=5):
'''
Execute a runner on the master and return the data from the runner
function
CLI Example:
.. code-block:: bash
salt publish.runner manage.down
'''
arg = _parse_args(arg)
load = {'cmd': 'minion_runner',
'fun': fun,
'arg': arg,
'tmo': timeout,
'id': __opts__['id']}
channel = salt.transport.client.ReqChannel.factory(__opts__)
try:
return channel.send(load)
except SaltReqTimeoutError:
return '\'{0}\' runner publish timed out'.format(fun)
finally:
channel.close()

128
salt/runners/manage.py
View file

@ -20,17 +20,16 @@ from salt.ext import six
from salt.ext.six.moves.urllib.request import urlopen as _urlopen # pylint: disable=no-name-in-module,import-error from salt.ext.six.moves.urllib.request import urlopen as _urlopen # pylint: disable=no-name-in-module,import-error
# Import salt libs # Import salt libs
import salt.client
import salt.client.ssh
import salt.key import salt.key
import salt.utils.compat import salt.utils.compat
import salt.utils.files import salt.utils.files
import salt.utils.minions import salt.utils.minions
import salt.utils.path import salt.utils.path
import salt.utils.raetevent import salt.utils.versions
import salt.client
import salt.client.ssh
import salt.wheel import salt.wheel
import salt.version import salt.version
from salt.utils.event import tagify
from salt.exceptions import SaltClientError, SaltSystemExit from salt.exceptions import SaltClientError, SaltSystemExit
FINGERPRINT_REGEX = re.compile(r'^([a-f0-9]{2}:){15}([a-f0-9]{2})$') FINGERPRINT_REGEX = re.compile(r'^([a-f0-9]{2}:){15}([a-f0-9]{2})$')
@ -218,7 +217,7 @@ def _show_ip_migration(show_ip, show_ipv4):
return show_ip return show_ip
def list_state(subset=None, show_ip=False, show_ipv4=None, state=None): def list_state(subset=None, show_ip=False, show_ipv4=None):
''' '''
.. versionadded:: 2015.8.0 .. versionadded:: 2015.8.0
.. versionchanged:: 2019.2.0 .. versionchanged:: 2019.2.0
@ -234,10 +233,6 @@ def list_state(subset=None, show_ip=False, show_ipv4=None, state=None):
show_ip : False show_ip : False
Also show the IP address each minion is connecting from. Also show the IP address each minion is connecting from.
state : 'available'
Show minions being in specific state that is one of 'available', 'joined',
'allowed', 'alived' or 'reaped'.
CLI Example: CLI Example:
.. code-block:: bash .. code-block:: bash
@ -245,30 +240,18 @@ def list_state(subset=None, show_ip=False, show_ipv4=None, state=None):
salt-run manage.list_state salt-run manage.list_state
''' '''
show_ip = _show_ip_migration(show_ip, show_ipv4) show_ip = _show_ip_migration(show_ip, show_ipv4)
conf_file = __opts__['conf_file']
opts = salt.config.client_config(conf_file) # Always return 'present' for 0MQ for now
if opts['transport'] == 'raet': # TODO: implement other states support for 0MQ
event = salt.utils.raetevent.PresenceEvent(__opts__, __opts__['sock_dir'], state=state) ckminions = salt.utils.minions.CkMinions(__opts__)
data = event.get_event(wait=60, tag=tagify('present', 'presence')) minions = ckminions.connected_ids(show_ip=show_ip, subset=subset)
key = 'present' if state is None else state
if not data or key not in data:
minions = []
else:
minions = data[key]
if subset:
minions = [m for m in minions if m in subset]
else:
# Always return 'present' for 0MQ for now
# TODO: implement other states support for 0MQ
ckminions = salt.utils.minions.CkMinions(__opts__)
minions = ckminions.connected_ids(show_ip=show_ip, subset=subset)
connected = dict(minions) if show_ip else sorted(minions) connected = dict(minions) if show_ip else sorted(minions)
return connected return connected
def list_not_state(subset=None, show_ip=False, show_ipv4=None, state=None): def list_not_state(subset=None, show_ip=False, show_ipv4=None):
''' '''
.. versionadded:: 2015.8.0 .. versionadded:: 2015.8.0
.. versionchanged:: 2019.2.0 .. versionchanged:: 2019.2.0
@ -284,10 +267,6 @@ def list_not_state(subset=None, show_ip=False, show_ipv4=None, state=None):
show_ip : False show_ip : False
Also show the IP address each minion is connecting from. Also show the IP address each minion is connecting from.
state : 'available'
Show minions being in specific state that is one of 'available', 'joined',
'allowed', 'alived' or 'reaped'.
CLI Example: CLI Example:
.. code-block:: bash .. code-block:: bash
@ -295,20 +274,13 @@ def list_not_state(subset=None, show_ip=False, show_ipv4=None, state=None):
salt-run manage.list_not_state salt-run manage.list_not_state
''' '''
show_ip = _show_ip_migration(show_ip, show_ipv4) show_ip = _show_ip_migration(show_ip, show_ipv4)
connected = list_state(subset=None, show_ip=show_ip, state=state) connected = list_state(subset=None, show_ip=show_ip)
key = salt.key.get_key(__opts__) key = salt.key.get_key(__opts__)
keys = key.list_keys() keys = key.list_keys()
# TODO: Need better way to handle key/node name difference for raet
# In raet case node name is '<name>_<kind>' meanwhile the key name
# is just '<name>'. So append '_minion' to the name to match.
appen_kind = isinstance(key, salt.key.RaetKey)
not_connected = [] not_connected = []
for minion in keys[key.ACC]: for minion in keys[key.ACC]:
if appen_kind:
minion += '_minion'
if minion not in connected and (subset is None or minion in subset): if minion not in connected and (subset is None or minion in subset):
not_connected.append(minion) not_connected.append(minion)
@ -389,7 +361,7 @@ def joined(subset=None, show_ip=False, show_ipv4=None):
salt-run manage.joined salt-run manage.joined
''' '''
show_ip = _show_ip_migration(show_ip, show_ipv4) show_ip = _show_ip_migration(show_ip, show_ipv4)
return list_state(subset=subset, show_ip=show_ip, state='joined') return list_state(subset=subset, show_ip=show_ip)
def not_joined(subset=None, show_ip=False, show_ipv4=None): def not_joined(subset=None, show_ip=False, show_ipv4=None):
@ -415,7 +387,7 @@ def not_joined(subset=None, show_ip=False, show_ipv4=None):
salt-run manage.not_joined salt-run manage.not_joined
''' '''
show_ip = _show_ip_migration(show_ip, show_ipv4) show_ip = _show_ip_migration(show_ip, show_ipv4)
return list_not_state(subset=subset, show_ip=show_ip, state='joined') return list_not_state(subset=subset, show_ip=show_ip)
def allowed(subset=None, show_ip=False, show_ipv4=None): def allowed(subset=None, show_ip=False, show_ipv4=None):
@ -441,7 +413,7 @@ def allowed(subset=None, show_ip=False, show_ipv4=None):
salt-run manage.allowed salt-run manage.allowed
''' '''
show_ip = _show_ip_migration(show_ip, show_ipv4) show_ip = _show_ip_migration(show_ip, show_ipv4)
return list_state(subset=subset, show_ip=show_ip, state='allowed') return list_state(subset=subset, show_ip=show_ip)
def not_allowed(subset=None, show_ip=False, show_ipv4=None): def not_allowed(subset=None, show_ip=False, show_ipv4=None):
@ -467,7 +439,7 @@ def not_allowed(subset=None, show_ip=False, show_ipv4=None):
salt-run manage.not_allowed salt-run manage.not_allowed
''' '''
show_ip = _show_ip_migration(show_ip, show_ipv4) show_ip = _show_ip_migration(show_ip, show_ipv4)
return list_not_state(subset=subset, show_ip=show_ip, state='allowed') return list_not_state(subset=subset, show_ip=show_ip)
def alived(subset=None, show_ip=False, show_ipv4=None): def alived(subset=None, show_ip=False, show_ipv4=None):
@ -493,7 +465,7 @@ def alived(subset=None, show_ip=False, show_ipv4=None):
salt-run manage.alived salt-run manage.alived
''' '''
show_ip = _show_ip_migration(show_ip, show_ipv4) show_ip = _show_ip_migration(show_ip, show_ipv4)
return list_state(subset=subset, show_ip=show_ip, state='alived') return list_state(subset=subset, show_ip=show_ip)
def not_alived(subset=None, show_ip=False, show_ipv4=None): def not_alived(subset=None, show_ip=False, show_ipv4=None):
@ -519,7 +491,7 @@ def not_alived(subset=None, show_ip=False, show_ipv4=None):
salt-run manage.not_alived salt-run manage.not_alived
''' '''
show_ip = _show_ip_migration(show_ip, show_ipv4) show_ip = _show_ip_migration(show_ip, show_ipv4)
return list_not_state(subset=subset, show_ip=show_ip, state='alived') return list_not_state(subset=subset, show_ip=show_ip)
def reaped(subset=None, show_ip=False, show_ipv4=None): def reaped(subset=None, show_ip=False, show_ipv4=None):
@ -545,7 +517,7 @@ def reaped(subset=None, show_ip=False, show_ipv4=None):
salt-run manage.reaped salt-run manage.reaped
''' '''
show_ip = _show_ip_migration(show_ip, show_ipv4) show_ip = _show_ip_migration(show_ip, show_ipv4)
return list_state(subset=subset, show_ip=show_ip, state='reaped') return list_state(subset=subset, show_ip=show_ip)
def not_reaped(subset=None, show_ip=False, show_ipv4=None): def not_reaped(subset=None, show_ip=False, show_ipv4=None):
@ -571,69 +543,7 @@ def not_reaped(subset=None, show_ip=False, show_ipv4=None):
salt-run manage.not_reaped salt-run manage.not_reaped
''' '''
show_ip = _show_ip_migration(show_ip, show_ipv4) show_ip = _show_ip_migration(show_ip, show_ipv4)
return list_not_state(subset=subset, show_ip=show_ip, state='reaped') return list_not_state(subset=subset, show_ip=show_ip)
def get_stats(estate=None, stack='road'):
'''
Print the stack stats
estate : None
The name of the target estate. Master stats would be requested by default
stack : 'road'
Show stats on either road or lane stack
Allowed values are 'road' or 'lane'.
CLI Example:
.. code-block:: bash
salt-run manage.get_stats [estate=alpha_minion] [stack=lane]
'''
conf_file = __opts__['conf_file']
opts = salt.config.client_config(conf_file)
if opts['transport'] == 'raet':
tag = tagify(stack, 'stats')
event = salt.utils.raetevent.StatsEvent(__opts__, __opts__['sock_dir'], tag=tag, estate=estate)
stats = event.get_event(wait=60, tag=tag)
else:
# TODO: implement 0MQ analog
stats = 'Not implemented'
return stats
def road_stats(estate=None):
'''
Print the estate road stack stats
estate : None
The name of the target estate. Master stats would be requested by default
CLI Example:
.. code-block:: bash
salt-run manage.road_stats [estate=alpha_minion]
'''
return get_stats(estate=estate, stack='road')
def lane_stats(estate=None):
'''
Print the estate manor lane stack stats
estate : None
The name of the target estate. Master stats would be requested by default
CLI Example:
.. code-block:: bash
salt-run manage.lane_stats [estate=alpha_minion]
'''
return get_stats(estate=estate, stack='lane')
def safe_accept(target, tgt_type='glob'): def safe_accept(target, tgt_type='glob'):

10
salt/transport/client.py
View file

@ -112,15 +112,12 @@ class AsyncReqChannel(AsyncChannel):
AsyncChannel._config_resolver() AsyncChannel._config_resolver()
import salt.transport.tcp import salt.transport.tcp
return salt.transport.tcp.AsyncTCPReqChannel(opts, **kwargs) return salt.transport.tcp.AsyncTCPReqChannel(opts, **kwargs)
elif ttype == 'raet':
import salt.transport.raet
return salt.transport.raet.RAETReqChannel(opts, **kwargs)
elif ttype == 'local': elif ttype == 'local':
import salt.transport.local import salt.transport.local
return salt.transport.local.AsyncLocalChannel(opts, **kwargs) return salt.transport.local.AsyncLocalChannel(opts, **kwargs)
else: else:
raise Exception( raise Exception(
'Channels are only defined for tcp, zeromq, raet, and local' 'Channels are only defined for tcp, zeromq, and local'
) )
# return NewKindOfChannel(opts, **kwargs) # return NewKindOfChannel(opts, **kwargs)
@ -166,15 +163,12 @@ class AsyncPubChannel(AsyncChannel):
AsyncChannel._config_resolver() AsyncChannel._config_resolver()
import salt.transport.tcp import salt.transport.tcp
return salt.transport.tcp.AsyncTCPPubChannel(opts, **kwargs) return salt.transport.tcp.AsyncTCPPubChannel(opts, **kwargs)
elif ttype == 'raet': # TODO:
import salt.transport.raet
return salt.transport.raet.AsyncRAETPubChannel(opts, **kwargs)
elif ttype == 'local': # TODO: elif ttype == 'local': # TODO:
import salt.transport.local import salt.transport.local
return salt.transport.local.AsyncLocalPubChannel(opts, **kwargs) return salt.transport.local.AsyncLocalPubChannel(opts, **kwargs)
else: else:
raise Exception( raise Exception(
'Channels are only defined for tcp, zeromq, raet, and local' 'Channels are only defined for tcp, zeromq, and local'
) )
# return NewKindOfChannel(opts, **kwargs) # return NewKindOfChannel(opts, **kwargs)

162
salt/transport/raet.py
View file

@ -1,162 +0,0 @@
# -*- coding: utf-8 -*-
'''
RAET transport classes
'''
from __future__ import absolute_import, print_function, unicode_literals
import time
# Import Salt Libs
import logging
import salt.utils.kinds as kinds
from salt.transport.client import ReqChannel
log = logging.getLogger(__name__)
try:
from raet import raeting, nacling
from raet.lane.stacking import LaneStack
from raet.lane.yarding import RemoteYard
except (ImportError, OSError):
# Don't die on missing transport libs since only one transport is required
pass
# Module globals for default LaneStack. Because RAETReqChannels are created on demand
# they do not have access to the master estate that motivated their creation
# Also in Raet a LaneStack can be shared shared by all channels in a given jobber
# For these reasons module globals are used to setup a shared jobber_stack as
# well has routing information for the master that motivated the jobber
# when a channel is not used in a jobber context then a LaneStack is created
# on demand.
jobber_stack = None # module global that holds raet jobber LaneStack
jobber_rxMsgs = {} # dict of deques one for each RAETReqChannel for the jobber
jobber_estate_name = None # module global of motivating master estate name
jobber_yard_name = None # module global of motivating master yard name
class RAETReqChannel(ReqChannel):
'''
Build the communication framework to communicate over the local process
uxd socket and send messages forwarded to the master. then wait for the
relative return message.
Two use cases:
mininion to master communication, normal use case
Minion is communicating via yard through minion Road to master
The destination route needs the estate name of the associated master
master call via runner, special use case
In the special case the master call external process is communicating
via a yard with the master manor yard
The destination route estate is None to indicate local estate
The difference between the two is how the destination route
is assigned.
'''
def __init__(self, opts, usage=None, **kwargs):
self.opts = opts
self.ttype = 'raet'
if usage == 'master_call': # runner.py master_call
self.dst = (None, None, 'local_cmd')
else: # everything else minion to master including salt-call
self.dst = (jobber_estate_name or None,
jobber_yard_name or None,
'remote_cmd')
self.stack = None
self.ryn = 'manor' # remote yard name
def __prep_stack(self):
'''
Prepare the stack objects
'''
global jobber_stack
if not self.stack:
if jobber_stack:
self.stack = jobber_stack
else:
self.stack = jobber_stack = self._setup_stack(ryn=self.ryn)
log.debug("RAETReqChannel Using Jobber Stack at = %s\n", self.stack.ha)
def _setup_stack(self, ryn='manor'):
'''
Setup and return the LaneStack and Yard used by by channel when global
not already setup such as in salt-call to communicate to-from the minion
'''
role = self.opts.get('id')
if not role:
emsg = ("Missing role(\'id\') required to setup RAETReqChannel.")
log.error(emsg + "\n")
raise ValueError(emsg)
kind = self.opts.get('__role') # application kind 'master', 'minion', etc
if kind not in kinds.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for RAETReqChannel.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.master],
kinds.APPL_KIND_NAMES[kinds.applKinds.syndic]]:
lanename = 'master'
elif kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.minion],
kinds.APPL_KIND_NAMES[kinds.applKinds.caller]]:
lanename = "{0}_{1}".format(role, kind)
else:
emsg = ("Unsupported application kind '{0}' for RAETReqChannel.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
name = 'channel' + nacling.uuid(size=18)
stack = LaneStack(name=name,
lanename=lanename,
sockdirpath=self.opts['sock_dir'])
stack.Pk = raeting.PackKind.pack
stack.addRemote(RemoteYard(stack=stack,
name=ryn,
lanename=lanename,
dirpath=self.opts['sock_dir']))
log.debug("Created Channel Jobber Stack %s\n", stack.name)
return stack
def crypted_transfer_decode_dictentry(self, load, dictkey=None, tries=3, timeout=60):
'''
We don't need to do the crypted_transfer_decode_dictentry routine for
raet, just wrap send.
'''
return self.send(load, tries, timeout)
def send(self, load, tries=3, timeout=60, raw=False):
'''
Send a message load and wait for a relative reply
One shot wonder
'''
self.__prep_stack()
tried = 1
start = time.time()
track = nacling.uuid(18)
src = (None, self.stack.local.name, track)
self.route = {'src': src, 'dst': self.dst}
msg = {'route': self.route, 'load': load}
self.stack.transmit(msg, self.stack.nameRemotes[self.ryn].uid)
while track not in jobber_rxMsgs:
self.stack.serviceAll()
while self.stack.rxMsgs:
msg, sender = self.stack.rxMsgs.popleft()
jobber_rxMsgs[msg['route']['dst'][2]] = msg
continue
if track in jobber_rxMsgs:
break
if time.time() - start > timeout:
if tried >= tries:
raise ValueError("Message send timed out after '{0} * {1}'"
" secs. route = {2} track = {3} load={4}".format(tries,
timeout,
self.route,
track,
load))
self.stack.transmit(msg, self.stack.nameRemotes['manor'].uid)
tried += 1
time.sleep(0.01)
return jobber_rxMsgs.pop(track).get('return', {})

10
salt/transport/server.py
View file

@ -31,9 +31,6 @@ class ReqServerChannel(object):
if ttype == 'zeromq': if ttype == 'zeromq':
import salt.transport.zeromq import salt.transport.zeromq
return salt.transport.zeromq.ZeroMQReqServerChannel(opts) return salt.transport.zeromq.ZeroMQReqServerChannel(opts)
elif ttype == 'raet':
import salt.transport.raet
return salt.transport.raet.RAETReqServerChannel(opts)
elif ttype == 'tcp': elif ttype == 'tcp':
import salt.transport.tcp import salt.transport.tcp
return salt.transport.tcp.TCPReqServerChannel(opts) return salt.transport.tcp.TCPReqServerChannel(opts)
@ -41,7 +38,7 @@ class ReqServerChannel(object):
import salt.transport.local import salt.transport.local
return salt.transport.local.LocalServerChannel(opts) return salt.transport.local.LocalServerChannel(opts)
else: else:
raise Exception('Channels are only defined for ZeroMQ and raet') raise Exception('Channels are only defined for ZeroMQ and TCP')
# return NewKindOfChannel(opts, **kwargs) # return NewKindOfChannel(opts, **kwargs)
def pre_fork(self, process_manager): def pre_fork(self, process_manager):
@ -79,9 +76,6 @@ class PubServerChannel(object):
if ttype == 'zeromq': if ttype == 'zeromq':
import salt.transport.zeromq import salt.transport.zeromq
return salt.transport.zeromq.ZeroMQPubServerChannel(opts, **kwargs) return salt.transport.zeromq.ZeroMQPubServerChannel(opts, **kwargs)
elif ttype == 'raet': # TODO:
import salt.transport.raet
return salt.transport.raet.RAETPubServerChannel(opts, **kwargs)
elif ttype == 'tcp': elif ttype == 'tcp':
import salt.transport.tcp import salt.transport.tcp
return salt.transport.tcp.TCPPubServerChannel(opts) return salt.transport.tcp.TCPPubServerChannel(opts)
@ -89,7 +83,7 @@ class PubServerChannel(object):
import salt.transport.local import salt.transport.local
return salt.transport.local.LocalPubServerChannel(opts, **kwargs) return salt.transport.local.LocalPubServerChannel(opts, **kwargs)
else: else:
raise Exception('Channels are only defined for ZeroMQ and raet') raise Exception('Channels are only defined for ZeroMQ and TCP')
# return NewKindOfChannel(opts, **kwargs) # return NewKindOfChannel(opts, **kwargs)
def pre_fork(self, process_manager, kwargs=None): def pre_fork(self, process_manager, kwargs=None):

40
salt/utils/event.py
View file

@ -128,27 +128,20 @@ def get_event(
''' '''
sock_dir = sock_dir or opts['sock_dir'] sock_dir = sock_dir or opts['sock_dir']
# TODO: AIO core is separate from transport # TODO: AIO core is separate from transport
if transport in ('zeromq', 'tcp'): if node == 'master':
if node == 'master': return MasterEvent(sock_dir,
return MasterEvent(sock_dir, opts,
opts, listen=listen,
listen=listen, io_loop=io_loop,
io_loop=io_loop, keep_loop=keep_loop,
keep_loop=keep_loop, raise_errors=raise_errors)
raise_errors=raise_errors) return SaltEvent(node,
return SaltEvent(node, sock_dir,
sock_dir, opts,
opts, listen=listen,
listen=listen, io_loop=io_loop,
io_loop=io_loop, keep_loop=keep_loop,
keep_loop=keep_loop, raise_errors=raise_errors)
raise_errors=raise_errors)
elif transport == 'raet':
import salt.utils.raetevent
return salt.utils.raetevent.RAETEvent(node,
sock_dir=sock_dir,
listen=listen,
opts=opts)
def get_master_event(opts, sock_dir, listen=True, io_loop=None, raise_errors=False): def get_master_event(opts, sock_dir, listen=True, io_loop=None, raise_errors=False):
@ -158,11 +151,6 @@ def get_master_event(opts, sock_dir, listen=True, io_loop=None, raise_errors=Fal
# TODO: AIO core is separate from transport # TODO: AIO core is separate from transport
if opts['transport'] in ('zeromq', 'tcp', 'detect'): if opts['transport'] in ('zeromq', 'tcp', 'detect'):
return MasterEvent(sock_dir, opts, listen=listen, io_loop=io_loop, raise_errors=raise_errors) return MasterEvent(sock_dir, opts, listen=listen, io_loop=io_loop, raise_errors=raise_errors)
elif opts['transport'] == 'raet':
import salt.utils.raetevent
return salt.utils.raetevent.MasterEvent(
opts=opts, sock_dir=sock_dir, listen=listen
)
def fire_args(opts, jid, tag_data, prefix=''): def fire_args(opts, jid, tag_data, prefix=''):

41
salt/utils/parsers.py
View file

@ -33,7 +33,6 @@ import salt.utils.args
import salt.utils.data import salt.utils.data
import salt.utils.files import salt.utils.files
import salt.utils.jid import salt.utils.jid
import salt.utils.kinds as kinds
import salt.utils.platform import salt.utils.platform
import salt.utils.process import salt.utils.process
import salt.utils.stringutils import salt.utils.stringutils
@ -2757,48 +2756,8 @@ class SaltCallOptionParser(six.with_metaclass(OptionParserMeta,
else: else:
opts = config.minion_config(self.get_config_file_path(), opts = config.minion_config(self.get_config_file_path(),
cache_minion_id=True) cache_minion_id=True)
if opts.get('transport') == 'raet':
if not self._find_raet_minion(opts): # must create caller minion
opts['__role'] = kinds.APPL_KIND_NAMES[kinds.applKinds.caller]
return opts return opts
def _find_raet_minion(self, opts):
'''
Returns true if local RAET Minion is available
'''
yardname = 'manor'
dirpath = opts['sock_dir']
role = opts.get('id')
if not role:
emsg = "Missing role required to setup RAET SaltCaller."
logger.error(emsg)
raise ValueError(emsg)
kind = opts.get('__role') # application kind 'master', 'minion', etc
if kind not in kinds.APPL_KINDS:
emsg = "Invalid application kind = '{0}' for RAET SaltCaller.".format(six.text_type(kind))
logger.error(emsg)
raise ValueError(emsg)
if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.minion], kinds.APPL_KIND_NAMES[kinds.applKinds.caller]]:
lanename = "{0}_{1}".format(role, kind)
else:
emsg = "Unsupported application kind '{0}' for RAET SaltCaller.".format(six.text_type(kind))
logger.error(emsg)
raise ValueError(emsg)
if kind == kinds.APPL_KIND_NAMES[kinds.applKinds.minion]: # minion check
try:
from raet.lane.yarding import Yard # pylint: disable=3rd-party-module-not-gated
ha, dirpath = Yard.computeHa(dirpath, lanename, yardname) # pylint: disable=invalid-name
if os.path.exists(ha) and not os.path.isfile(ha) and not os.path.isdir(ha): # minion manor yard
return True
except ImportError as ex:
logger.error("Error while importing Yard: %s", ex)
return False
def process_module_dirs(self): def process_module_dirs(self):
for module_dir in self.options.module_dirs: for module_dir in self.options.module_dirs:
# Provide some backwards compatibility with previous comma # Provide some backwards compatibility with previous comma

340
salt/utils/raetevent.py
View file

@ -1,340 +0,0 @@
# -*- coding: utf-8 -*-
'''
Manage events
This module is used to manage events via RAET
'''
# pylint: disable=3rd-party-module-not-gated
# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
import os
import logging
import time
from collections import MutableMapping
# Import salt libs
import salt.payload
import salt.loader
import salt.state
import salt.utils.event
import salt.utils.kinds as kinds
from salt import transport
from salt import syspaths
try:
from raet import raeting, nacling
from raet.lane.stacking import LaneStack
from raet.lane.yarding import RemoteYard
HAS_RAET = True
except ImportError:
HAS_RAET = False
# Import 3rd-party libs
from salt.ext import six
log = logging.getLogger(__name__)
def __virtual__():
return HAS_RAET
class RAETEvent(object):
'''
The base class used to manage salt events
'''
def __init__(self, node, sock_dir=None, listen=True, opts=None):
'''
Set up the stack and remote yard
'''
self.node = node # application kind see kinds.APPL_KIND_NAMES
self.sock_dir = sock_dir
if opts is None:
opts = {}
self.opts = opts
self.stack = None
self.ryn = 'manor' # remote yard name
self.connected = False
self.cpub = False
self.__load_cache_regex()
self.__prep_stack(listen)
@classmethod
def __load_cache_regex(cls):
'''
Initialize the regular expression cache and put it in the
class namespace. The regex search strings will be prepend with '^'
'''
# This is in the class namespace, to minimize cache memory
# usage and maximize cache hits
# The prepend='^' is to reduce differences in behavior between
# the default 'startswith' and the optional 'regex' match_type
cls.cache_regex = salt.utils.cache.CacheRegex(prepend='^')
def __prep_stack(self, listen):
'''
Prepare the stack objects
'''
if not self.stack:
if hasattr(transport, 'jobber_stack') and transport.jobber_stack:
self.stack = transport.jobber_stack
else:
self.stack = transport.jobber_stack = self._setup_stack(ryn=self.ryn)
log.debug("RAETEvent Using Jobber Stack at = %s\n", self.stack.ha)
if listen:
self.subscribe()
def _setup_stack(self, ryn='manor'):
kind = self.opts.get('__role', '') # opts optional for master
if kind: # not all uses of Raet SaltEvent has opts defined
if kind not in kinds.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for RAET SaltEvent.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind != self.node:
emsg = ("Mismatch between node = '{0}' and kind = '{1}' in "
"RAET SaltEvent.".format(self.node, kind))
log.error(emsg + '\n')
raise ValueError(emsg)
if self.node in [kinds.APPL_KIND_NAMES[kinds.applKinds.master],
kinds.APPL_KIND_NAMES[kinds.applKinds.syndic]]: # []'master', 'syndic']
lanename = 'master'
elif self.node in [kinds.APPL_KIND_NAMES[kinds.applKinds.minion],
kinds.APPL_KIND_NAMES[kinds.applKinds.caller]]: # ['minion', 'caller']
role = self.opts.get('id', '') # opts required for minion
if not role:
emsg = ("Missing role required to setup RAET SaltEvent.")
log.error(emsg + "\n")
raise ValueError(emsg)
if not kind:
emsg = "Missing kind required to setup RAET SaltEvent."
log.error(emsg + '\n')
raise ValueError(emsg)
lanename = "{0}_{1}".format(role, kind)
else:
emsg = ("Unsupported application node kind '{0}' for RAET SaltEvent.".format(self.node))
log.error(emsg + '\n')
raise ValueError(emsg)
name = 'event' + nacling.uuid(size=18)
cachedir = self.opts.get('cachedir', os.path.join(syspaths.CACHE_DIR, self.node))
stack = LaneStack(
name=name,
lanename=lanename,
sockdirpath=self.sock_dir)
stack.Pk = raeting.PackKind.pack.value
stack.addRemote(RemoteYard(stack=stack,
lanename=lanename,
name=ryn,
dirpath=self.sock_dir))
return stack
def subscribe(self, tag=None, match_type=None):
'''
Included for compat with zeromq events, not required
'''
if not self.connected:
self.connect_pub()
def unsubscribe(self, tag=None, match_type=None):
'''
Included for compat with zeromq events, not required
'''
return
def connect_pub(self):
'''
Establish the publish connection
'''
try:
route = {'dst': (None, self.ryn, 'event_req'),
'src': (None, self.stack.local.name, None)}
msg = {'route': route}
self.stack.transmit(msg, self.stack.nameRemotes[self.ryn].uid)
self.stack.serviceAll()
self.connected = True
self.cpub = True
except Exception:
pass
def connect_pull(self, timeout=1000):
'''
Included for compat with zeromq events, not required
'''
return
@classmethod
def unpack(cls, raw, serial=None):
'''
Included for compat with zeromq events, not required
'''
return raw
def get_event(self, wait=5, tag='', match_type=None, full=False, no_block=None,
auto_reconnect=False):
'''
Get a single publication.
IF no publication available THEN block for up to wait seconds
AND either return publication OR None IF no publication available.
IF wait is 0 then block forever.
'''
if not self.connected:
self.connect_pub()
start = time.time()
while True:
self.stack.serviceAll()
if self.stack.rxMsgs:
msg, sender = self.stack.rxMsgs.popleft()
if 'tag' not in msg and 'data' not in msg:
# Invalid event, how did this get here?
continue
if not msg['tag'].startswith(tag) and self.cache_regex.get(tag).search(msg['tag']) is None:
# Not what we are looking for, throw it away
continue
if full:
return msg
else:
return msg['data']
if start + wait < time.time():
return None
time.sleep(0.01)
def get_event_noblock(self):
'''
Get the raw event msg without blocking or any other niceties
'''
if not self.connected:
self.connect_pub()
self.stack.serviceAll()
if self.stack.rxMsgs:
msg, sender = self.stack.rxMsgs.popleft()
if 'tag' not in msg and 'data' not in msg:
# Invalid event, how did this get here?
return None
return msg
def iter_events(self, tag='', full=False, auto_reconnect=False):
'''
Creates a generator that continuously listens for events
'''
while True:
data = self.get_event(tag=tag, full=full, auto_reconnect=auto_reconnect)
if data is None:
continue
yield data
def fire_event(self, data, tag, timeout=1000):
'''
Send a single event into the publisher with paylod dict "data" and event
identifier "tag"
'''
# Timeout is retained for compat with zeromq events
if not six.text_type(tag): # no empty tags allowed
raise ValueError('Empty tag.')
if not isinstance(data, MutableMapping): # data must be dict
raise ValueError('Dict object expected, not \'{0}\'.'.format(data))
route = {'dst': (None, self.ryn, 'event_fire'),
'src': (None, self.stack.local.name, None)}
msg = {'route': route, 'tag': tag, 'data': data}
self.stack.transmit(msg, self.stack.nameRemotes[self.ryn].uid)
self.stack.serviceAll()
def fire_ret_load(self, load):
'''
Fire events based on information in the return load
'''
if load.get('retcode') and load.get('fun'):
# Minion fired a bad retcode, fire an event
if load['fun'] in salt.utils.event.SUB_EVENT:
try:
for tag, data in six.iteritems(load.get('return', {})):
data['retcode'] = load['retcode']
tags = tag.split('_|-')
if data.get('result') is False:
self.fire_event(
data,
'{0}.{1}'.format(tags[0], tags[-1])) # old dup event
data['jid'] = load['jid']
data['id'] = load['id']
data['success'] = False
data['return'] = 'Error: {0}.{1}'.format(tags[0], tags[-1])
data['fun'] = load['fun']
data['user'] = load['user']
self.fire_event(
data,
salt.utils.event.tagify([load['jid'],
'sub',
load['id'],
'error',
load['fun']],
'job'))
except Exception:
pass
def close_pub(self):
'''
Here for compatability
'''
return
def destroy(self):
if hasattr(self, 'stack'):
self.stack.server.close()
class MasterEvent(RAETEvent):
'''
Create a master event management object
'''
def __init__(self, opts, sock_dir, listen=True):
super(MasterEvent, self).__init__('master', opts=opts, sock_dir=sock_dir, listen=listen)
class PresenceEvent(MasterEvent):
def __init__(self, opts, sock_dir, listen=True, state=None):
self.state = state
super(PresenceEvent, self).__init__(opts=opts, sock_dir=sock_dir, listen=listen)
def connect_pub(self):
'''
Establish the publish connection
'''
try:
route = {'dst': (None, self.ryn, 'presence_req'),
'src': (None, self.stack.local.name, None)}
msg = {'route': route}
if self.state:
msg['data'] = {'state': self.state}
self.stack.transmit(msg, self.stack.nameRemotes[self.ryn].uid)
self.stack.serviceAll()
self.connected = True
except Exception:
pass
class StatsEvent(MasterEvent):
def __init__(self, opts, sock_dir, tag, estate=None, listen=True):
super(StatsEvent, self).__init__(opts=opts, sock_dir=sock_dir, listen=listen)
self.tag = tag
self.estate = estate
def connect_pub(self):
'''
Establish the publish connection
'''
try:
route = {'dst': (self.estate, None, 'stats_req'),
'src': (None, self.stack.local.name, None)}
msg = {'route': route, 'tag': self.tag}
self.stack.transmit(msg, self.stack.nameRemotes[self.ryn].uid)
self.stack.serviceAll()
self.connected = True
except Exception:
pass

181
salt/utils/raetlane.py
View file

@ -1,181 +0,0 @@
# -*- coding: utf-8 -*-
'''
Provides RAET LaneStack interface for interprocess communications in Salt Raet
to a remote yard, default name for remote is 'manor' .
Usages are for RAETChannels and RAETEvents
This provides a single module global LaneStack to be shared by all users in
the same process. This combines into one stack the channel and event bus.
The module attributes:
lane_rx_msgs
is a dict of deques keyed by the destination share name
recipients each value deque holds messages that were addressed
to that share name
lane_stack
is the shared LaneStack object
lane_estate_name
is the motivating master estate name when applicable
lane_yard_name
is the motivating master yard name when applicable
Because RaetChannels are created on demand
they do not have access to the master estate that motivated their creation
the module globals lane_estate_name and lane_yard_name are provided to setup
so that channels using the routing information for the master that motivated the jobber
when a channel is not used in a jobber context then a LaneStack is created
on demand.
Example usage:
import raetlane
raetlane.prep()
track = nacling.uuid(18)
src = (None, 'localyardname', track)
dst = ('remotestackname', 'remoteyardname', 'remotesharename')
route = {'src': src, 'dst': dst}
msg = {'route': route, 'body': {}}
raetlane.transmit(msg)
raetlane.service()
msg = raetlane.wait(share=track, timeout=5.0)
if not msg:
raise ValueError("Timed out out waiting for response")
'''
# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
try:
from raet import raeting, nacling
from raet.lane.stacking import LaneStack
from raet.lane.yarding import RemoteYard
HAS_RAET = True
except ImportError:
HAS_RAET = False
if HAS_RAET:
# pylint: disable=3rd-party-module-not-gated
import time
# Import Salt Libs
import logging
import salt.utils.kinds as kinds
log = logging.getLogger(__name__)
# Module globals for default shared LaneStack for a process.
rx_msgs = {} # module global dict of deques one for each receipient of msgs
lane_stack = None # module global that holds raet LaneStack
remote_yard = None # module global that holds raet remote Yard
master_estate_name = None # module global of motivating master estate name
master_yard_name = None # module global of motivating master yard name
def prep(opts, ryn='manor'):
'''
required items in opts are keys
'id'
'__role'
'sock_dir'
ryn is the remote yard name to communicate with
each use much call raetlane.prep() to ensure lanestack is setup
'''
if not lane_stack:
_setup(opts=opts, ryn=ryn)
def _setup(opts, ryn='manor'):
'''
Setup the LaneStack lane_stack and RemoteYard lane_remote_yard global
'''
global lane_stack, remote_yard # pylint: disable=W0602
role = opts.get('id')
if not role:
emsg = ("Missing role required to setup LaneStack.")
log.error(emsg + "\n")
raise ValueError(emsg)
kind = opts.get('__role') # application kind 'master', 'minion', etc
if kind not in kinds.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for LaneStack.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.master],
kinds.APPL_KIND_NAMES[kinds.applKinds.syndic]]:
lanename = 'master'
elif kind == [kinds.APPL_KIND_NAMES[kinds.applKinds.minion],
kinds.APPL_KIND_NAMES[kinds.applKinds.caller]]:
lanename = "{0}_{1}".format(role, kind)
else:
emsg = ("Unsupported application kind '{0}' for LaneStack.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
name = 'lanestack' + nacling.uuid(size=18)
lane_stack = LaneStack(name=name,
lanename=lanename,
sockdirpath=opts['sock_dir'])
lane_stack.Pk = raeting.PackKind.pack.value
log.debug(
'Created new LaneStack and local Yard named %s at %s\n',
lane_stack.name, lane_stack.ha
)
remote_yard = RemoteYard(stack=lane_stack,
name=ryn,
lanename=lanename,
dirpath=opts['sock_dir'])
lane_stack.addRemote(remote_yard)
log.debug(
'Added to LaneStack %s remote Yard named %s at %s\n',
lane_stack.name, remote_yard.name, remote_yard.ha
)
def transmit(msg):
'''
Sends msg to remote_yard
'''
lane_stack.transmit(msg, remote_yard.uid)
def service():
'''
Service the lane_stack and move any received messages into their associated
deques in rx_msgs keyed by the destination share in the msg route dict
'''
lane_stack.serviceAll()
while lane_stack.rxMsgs:
msg, sender = lane_stack.rxMsgs.popleft()
rx_msgs[msg['route']['dst'][2]] = msg
def receive(share):
'''
Returns first message from deque at key given by share in rx_msgs if any
otherwise returns None
'''
service()
if share in rx_msgs:
if rx_msgs[share]:
return rx_msgs[share].popleft()
return None
def wait(share, timeout=0.0, delay=0.01):
'''
Blocks until receives a msg addressed to share or timeout
Return msg or None if timed out
Delay is sleep time between services
'''
start = time.time()
while True:
msg = receive(share)
if msg:
return msg
time.sleep(delay)
if timeout > 0.0 and (time.time() - start) >= timeout:
return None

11
salt/utils/yamldumper.py
View file

@ -20,13 +20,6 @@ import collections
import salt.utils.context import salt.utils.context
from salt.utils.odict import OrderedDict from salt.utils.odict import OrderedDict
try:
from ioflo.aid.odicting import odict # pylint: disable=E0611
HAS_IOFLO = True
except ImportError:
odict = None
HAS_IOFLO = False
__all__ = ['OrderedDumper', 'SafeOrderedDumper', 'IndentedSafeOrderedDumper', __all__ = ['OrderedDumper', 'SafeOrderedDumper', 'IndentedSafeOrderedDumper',
'get_dumper', 'dump', 'safe_dump'] 'get_dumper', 'dump', 'safe_dump']
@ -93,10 +86,6 @@ SafeOrderedDumper.add_representer(
'tag:yaml.org,2002:timestamp', 'tag:yaml.org,2002:timestamp',
SafeOrderedDumper.represent_scalar) SafeOrderedDumper.represent_scalar)
if HAS_IOFLO:
OrderedDumper.add_representer(odict, represent_ordereddict)
SafeOrderedDumper.add_representer(odict, represent_ordereddict)
def get_dumper(dumper_name): def get_dumper(dumper_name):
return { return {

3
salt/version.py
View file

@ -583,11 +583,8 @@ def dependency_information(include_salt_cloud=False):
('msgpack-pure', 'msgpack_pure', 'version'), ('msgpack-pure', 'msgpack_pure', 'version'),
('pycrypto', 'Crypto', '__version__'), ('pycrypto', 'Crypto', '__version__'),
('pycryptodome', 'Cryptodome', 'version_info'), ('pycryptodome', 'Cryptodome', 'version_info'),
('libnacl', 'libnacl', '__version__'),
('PyYAML', 'yaml', '__version__'), ('PyYAML', 'yaml', '__version__'),
('ioflo', 'ioflo', '__version__'),
('PyZMQ', 'zmq', '__version__'), ('PyZMQ', 'zmq', '__version__'),
('RAET', 'raet', '__version__'),
('ZMQ', 'zmq', 'zmq_version'), ('ZMQ', 'zmq', 'zmq_version'),
('Mako', 'mako', '__version__'), ('Mako', 'mako', '__version__'),
('Tornado', 'tornado', 'version'), ('Tornado', 'tornado', 'version'),

31
setup.py
View file

@ -14,7 +14,6 @@ from __future__ import absolute_import, print_function, with_statement
import os import os
import sys import sys
import glob import glob
import time
import operator import operator
import platform import platform
try: try:
@ -124,7 +123,6 @@ SALT_VERSION_HARDCODED = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', '_
SALT_SYSPATHS_HARDCODED = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', '_syspaths.py') SALT_SYSPATHS_HARDCODED = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', '_syspaths.py')
SALT_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'base.txt') SALT_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'base.txt')
SALT_ZEROMQ_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'zeromq.txt') SALT_ZEROMQ_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'zeromq.txt')
SALT_RAET_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'raet.txt')
SALT_WINDOWS_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'pkg', 'windows', 'req.txt') SALT_WINDOWS_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'pkg', 'windows', 'req.txt')
SALT_LONG_DESCRIPTION_FILE = os.path.join(os.path.abspath(SETUP_DIRNAME), 'README.rst') SALT_LONG_DESCRIPTION_FILE = os.path.join(os.path.abspath(SETUP_DIRNAME), 'README.rst')
@ -437,7 +435,7 @@ class DownloadWindowsDlls(Command):
url = 'https://repo.saltstack.com/windows/dependencies/{bits}/{fname}.dll' url = 'https://repo.saltstack.com/windows/dependencies/{bits}/{fname}.dll'
dest = os.path.join(os.path.dirname(sys.executable), '{fname}.dll') dest = os.path.join(os.path.dirname(sys.executable), '{fname}.dll')
with indent_log(): with indent_log():
for fname in ('libeay32', 'libsodium', 'ssleay32', 'msvcr120'): for fname in ('libeay32', 'ssleay32', 'msvcr120'):
# See if the library is already on the system # See if the library is already on the system
if find_library(fname): if find_library(fname):
continue continue
@ -795,8 +793,9 @@ class SaltDistribution(distutils.dist.Distribution):
''' '''
global_options = distutils.dist.Distribution.global_options + [ global_options = distutils.dist.Distribution.global_options + [
('ssh-packaging', None, 'Run in SSH packaging mode'), ('ssh-packaging', None, 'Run in SSH packaging mode'),
('salt-transport=', None, 'The transport to prepare salt for. Choices are \'zeromq\' ' ('salt-transport=', None, 'The transport to prepare salt for. Currently, the only choice '
'\'raet\' or \'both\'. Defaults to \'zeromq\'', 'zeromq')] + [ 'is \'zeromq\'. This may be expanded in the future. Defaults to '
'\'zeromq\'', 'zeromq')] + [
('with-salt-version=', None, 'Set a fixed version for Salt instead calculating it'), ('with-salt-version=', None, 'Set a fixed version for Salt instead calculating it'),
# Salt's Paths Configuration Settings # Salt's Paths Configuration Settings
('salt-root-dir=', None, ('salt-root-dir=', None,
@ -1009,19 +1008,11 @@ class SaltDistribution(distutils.dist.Distribution):
if self.salt_transport == 'zeromq': if self.salt_transport == 'zeromq':
install_requires += _parse_requirements_file(SALT_ZEROMQ_REQS) install_requires += _parse_requirements_file(SALT_ZEROMQ_REQS)
elif self.salt_transport == 'raet':
install_requires += _parse_requirements_file(SALT_RAET_REQS)
if IS_WINDOWS_PLATFORM: if IS_WINDOWS_PLATFORM:
install_requires = _parse_requirements_file(SALT_WINDOWS_REQS) install_requires = _parse_requirements_file(SALT_WINDOWS_REQS)
return install_requires return install_requires
@property
def _property_extras_require(self):
if self.ssh_packaging:
return {}
return {'RAET': _parse_requirements_file(SALT_RAET_REQS)}
@property @property
def _property_scripts(self): def _property_scripts(self):
# Scripts common to all scenarios # Scripts common to all scenarios
@ -1189,15 +1180,7 @@ class SaltDistribution(distutils.dist.Distribution):
freezer_includes.append(str(os.path.basename(mod.identifier))) freezer_includes.append(str(os.path.basename(mod.identifier)))
except ImportError: except ImportError:
pass pass
# Include C extension that convinces esky to package up the libsodium C library
# This is needed for ctypes to find it in libnacl which is in turn needed for raet
# see pkg/smartos/esky/sodium_grabber{.c,_installer.py}
freezer_includes.extend([
'sodium_grabber',
'ioflo',
'raet',
'libnacl',
])
return freezer_includes return freezer_includes
# <---- Esky Setup ----------------------------------------------------------------------------------------------- # <---- Esky Setup -----------------------------------------------------------------------------------------------
@ -1214,10 +1197,10 @@ class SaltDistribution(distutils.dist.Distribution):
elif self.salt_transport is None: elif self.salt_transport is None:
self.salt_transport = 'zeromq' self.salt_transport = 'zeromq'
if self.salt_transport not in ('zeromq', 'raet', 'both', 'ssh', 'none'): if self.salt_transport not in ('zeromq', 'both', 'ssh', 'none'):
raise DistutilsArgError( raise DistutilsArgError(
'The value of --salt-transport needs be \'zeromq\', ' 'The value of --salt-transport needs be \'zeromq\', '
'\'raet\', \'both\', \'ssh\' or \'none\' not \'{0}\''.format( '\'both\', \'ssh\', or \'none\' not \'{0}\''.format(
self.salt_transport self.salt_transport
) )
) )

4
tests/conftest.py
View file

@ -88,9 +88,9 @@ def pytest_addoption(parser):
parser.addoption( parser.addoption(
'--transport', '--transport',
default='zeromq', default='zeromq',
choices=('zeromq', 'raet', 'tcp'), choices=('zeromq', 'tcp'),
help=('Select which transport to run the integration tests with, ' help=('Select which transport to run the integration tests with, '
'zeromq, raet, or tcp. Default: %default') 'zeromq or tcp. Default: %default')
) )
test_selection_group = parser.getgroup('Tests Selection') test_selection_group = parser.getgroup('Tests Selection')
test_selection_group.addoption( test_selection_group.addoption(

54
tests/integration/__init__.py
View file

@ -41,9 +41,11 @@ from tests.support.mixins import CheckShellBinaryNameAndVersionMixin, ShellCaseC
from tests.support.mixins import AdaptedConfigurationTestCaseMixin, SaltClientTestCaseMixin from tests.support.mixins import AdaptedConfigurationTestCaseMixin, SaltClientTestCaseMixin
from tests.support.mixins import SaltMinionEventAssertsMixin, SaltReturnAssertsMixin from tests.support.mixins import SaltMinionEventAssertsMixin, SaltReturnAssertsMixin
from tests.support.runtests import RUNTIME_VARS from tests.support.runtests import RUNTIME_VARS
# Import Salt libs # Import Salt libs
import salt import salt
import salt.config import salt.config
import salt.master
import salt.minion import salt.minion
import salt.runner import salt.runner
import salt.output import salt.output
@ -58,28 +60,17 @@ import salt.utils.yaml
import salt.log.setup as salt_log_setup import salt.log.setup as salt_log_setup
from salt.utils.verify import verify_env from salt.utils.verify import verify_env
from salt.utils.immutabletypes import freeze from salt.utils.immutabletypes import freeze
from salt.utils.nb_popen import NonBlockingPopen
from salt.exceptions import SaltClientError from salt.exceptions import SaltClientError
try:
import salt.master
except ImportError:
# Not required for raet tests
pass
# Import 3rd-party libs # Import 3rd-party libs
import msgpack import msgpack
from salt.ext import six from salt.ext import six
from salt.ext.six.moves import cStringIO
try: try:
import salt.ext.six.moves.socketserver as socketserver import salt.ext.six.moves.socketserver as socketserver
except ImportError: except ImportError:
import socketserver import socketserver
from tornado import gen
from tornado import ioloop
# Import salt tests support libs # Import salt tests support libs
from tests.support.processes import SaltMaster, SaltMinion, SaltSyndic from tests.support.processes import SaltMaster, SaltMinion, SaltSyndic
@ -213,8 +204,6 @@ class TestDaemon(object):
if self.parser.options.transport == 'zeromq': if self.parser.options.transport == 'zeromq':
self.start_zeromq_daemons() self.start_zeromq_daemons()
elif self.parser.options.transport == 'raet':
self.start_raet_daemons()
elif self.parser.options.transport == 'tcp': elif self.parser.options.transport == 'tcp':
self.start_tcp_daemons() self.start_tcp_daemons()
@ -506,31 +495,6 @@ class TestDaemon(object):
sys.stdout.flush() sys.stdout.flush()
raise TestDaemonStartFailed() raise TestDaemonStartFailed()
def start_raet_daemons(self):
'''
Fire up the raet daemons!
'''
import salt.daemons.flo
self.master_process = self.start_daemon(salt.daemons.flo.IofloMaster,
self.master_opts,
'start')
self.minion_process = self.start_daemon(salt.daemons.flo.IofloMinion,
self.minion_opts,
'tune_in')
self.sub_minion_process = self.start_daemon(salt.daemons.flo.IofloMinion,
self.sub_minion_opts,
'tune_in')
# Wait for the daemons to all spin up
time.sleep(5)
# self.smaster_process = self.start_daemon(salt.daemons.flo.IofloMaster,
# self.syndic_master_opts,
# 'start')
# no raet syndic daemon yet
start_tcp_daemons = start_zeromq_daemons start_tcp_daemons = start_zeromq_daemons
def prep_syndic(self): def prep_syndic(self):
@ -851,15 +815,6 @@ class TestDaemon(object):
proxy_opts['hosts.file'] = os.path.join(TMP, 'rootdir-proxy', 'hosts') proxy_opts['hosts.file'] = os.path.join(TMP, 'rootdir-proxy', 'hosts')
proxy_opts['aliases.file'] = os.path.join(TMP, 'rootdir-proxy', 'aliases') proxy_opts['aliases.file'] = os.path.join(TMP, 'rootdir-proxy', 'aliases')
if transport == 'raet':
master_opts['transport'] = 'raet'
master_opts['raet_port'] = 64506
minion_opts['transport'] = 'raet'
minion_opts['raet_port'] = 64510
sub_minion_opts['transport'] = 'raet'
sub_minion_opts['raet_port'] = 64520
# syndic_master_opts['transport'] = 'raet'
if transport == 'tcp': if transport == 'tcp':
master_opts['transport'] = 'tcp' master_opts['transport'] = 'tcp'
minion_opts['transport'] = 'tcp' minion_opts['transport'] = 'tcp'
@ -1047,13 +1002,11 @@ class TestDaemon(object):
os.path.join(master_opts['pki_dir'], 'minions_rejected'), os.path.join(master_opts['pki_dir'], 'minions_rejected'),
os.path.join(master_opts['pki_dir'], 'minions_denied'), os.path.join(master_opts['pki_dir'], 'minions_denied'),
os.path.join(master_opts['cachedir'], 'jobs'), os.path.join(master_opts['cachedir'], 'jobs'),
os.path.join(master_opts['cachedir'], 'raet'),
os.path.join(master_opts['root_dir'], 'cache', 'tokens'), os.path.join(master_opts['root_dir'], 'cache', 'tokens'),
os.path.join(syndic_master_opts['pki_dir'], 'minions'), os.path.join(syndic_master_opts['pki_dir'], 'minions'),
os.path.join(syndic_master_opts['pki_dir'], 'minions_pre'), os.path.join(syndic_master_opts['pki_dir'], 'minions_pre'),
os.path.join(syndic_master_opts['pki_dir'], 'minions_rejected'), os.path.join(syndic_master_opts['pki_dir'], 'minions_rejected'),
os.path.join(syndic_master_opts['cachedir'], 'jobs'), os.path.join(syndic_master_opts['cachedir'], 'jobs'),
os.path.join(syndic_master_opts['cachedir'], 'raet'),
os.path.join(syndic_master_opts['root_dir'], 'cache', 'tokens'), os.path.join(syndic_master_opts['root_dir'], 'cache', 'tokens'),
os.path.join(master_opts['pki_dir'], 'accepted'), os.path.join(master_opts['pki_dir'], 'accepted'),
os.path.join(master_opts['pki_dir'], 'rejected'), os.path.join(master_opts['pki_dir'], 'rejected'),
@ -1061,16 +1014,13 @@ class TestDaemon(object):
os.path.join(syndic_master_opts['pki_dir'], 'accepted'), os.path.join(syndic_master_opts['pki_dir'], 'accepted'),
os.path.join(syndic_master_opts['pki_dir'], 'rejected'), os.path.join(syndic_master_opts['pki_dir'], 'rejected'),
os.path.join(syndic_master_opts['pki_dir'], 'pending'), os.path.join(syndic_master_opts['pki_dir'], 'pending'),
os.path.join(syndic_master_opts['cachedir'], 'raet'),
os.path.join(minion_opts['pki_dir'], 'accepted'), os.path.join(minion_opts['pki_dir'], 'accepted'),
os.path.join(minion_opts['pki_dir'], 'rejected'), os.path.join(minion_opts['pki_dir'], 'rejected'),
os.path.join(minion_opts['pki_dir'], 'pending'), os.path.join(minion_opts['pki_dir'], 'pending'),
os.path.join(minion_opts['cachedir'], 'raet'),
os.path.join(sub_minion_opts['pki_dir'], 'accepted'), os.path.join(sub_minion_opts['pki_dir'], 'accepted'),
os.path.join(sub_minion_opts['pki_dir'], 'rejected'), os.path.join(sub_minion_opts['pki_dir'], 'rejected'),
os.path.join(sub_minion_opts['pki_dir'], 'pending'), os.path.join(sub_minion_opts['pki_dir'], 'pending'),
os.path.join(sub_minion_opts['cachedir'], 'raet'),
os.path.dirname(master_opts['log_file']), os.path.dirname(master_opts['log_file']),
minion_opts['extension_modules'], minion_opts['extension_modules'],
sub_minion_opts['extension_modules'], sub_minion_opts['extension_modules'],

22
tests/integration/shell/test_key.py
View file

@ -115,14 +115,6 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin):
'Unaccepted Keys:', 'Unaccepted Keys:',
'Rejected Keys:' 'Rejected Keys:'
] ]
elif self.master_opts['transport'] == 'raet':
expect = [
'Accepted Keys:',
'minion',
'sub_minion',
'Unaccepted Keys:',
'Rejected Keys:'
]
self.assertEqual(data, expect) self.assertEqual(data, expect)
def test_list_json_out(self): def test_list_json_out(self):
@ -143,10 +135,6 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin):
'minions_denied': [], 'minions_denied': [],
'minions_pre': [], 'minions_pre': [],
'minions': ['minion', 'sub_minion']} 'minions': ['minion', 'sub_minion']}
elif self.master_opts['transport'] == 'raet':
expect = {'accepted': ['minion', 'sub_minion'],
'rejected': [],
'pending': []}
self.assertEqual(ret, expect) self.assertEqual(ret, expect)
def test_list_yaml_out(self): def test_list_yaml_out(self):
@ -167,10 +155,6 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin):
'minions_denied': [], 'minions_denied': [],
'minions_pre': [], 'minions_pre': [],
'minions': ['minion', 'sub_minion']} 'minions': ['minion', 'sub_minion']}
elif self.master_opts['transport'] == 'raet':
expect = {'accepted': ['minion', 'sub_minion'],
'rejected': [],
'pending': []}
self.assertEqual(ret, expect) self.assertEqual(ret, expect)
def test_list_raw_out(self): def test_list_raw_out(self):
@ -193,10 +177,6 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin):
'minions_denied': [], 'minions_denied': [],
'minions_pre': [], 'minions_pre': [],
'minions': ['minion', 'sub_minion']} 'minions': ['minion', 'sub_minion']}
elif self.master_opts['transport'] == 'raet':
expect = {'accepted': ['minion', 'sub_minion'],
'rejected': [],
'pending': []}
self.assertEqual(ret, expect) self.assertEqual(ret, expect)
def test_list_acc(self): def test_list_acc(self):
@ -251,8 +231,6 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin):
key_names = None key_names = None
if self.master_opts['transport'] in ('zeromq', 'tcp'): if self.master_opts['transport'] in ('zeromq', 'tcp'):
key_names = ('minibar.pub', 'minibar.pem') key_names = ('minibar.pub', 'minibar.pem')
elif self.master_opts['transport'] == 'raet':
key_names = ('minibar.key',)
for fname in key_names: for fname in key_names:
self.assertTrue(os.path.isfile(os.path.join(tempdir, fname))) self.assertTrue(os.path.isfile(os.path.join(tempdir, fname)))
finally: finally:

2
tests/integration/shell/test_matcher.py
View file

@ -229,8 +229,6 @@ class MatchTest(ShellCase, ShellCaseCommonTestsMixin):
'No command was sent, no jid was ' 'No command was sent, no jid was '
'assigned.' 'assigned.'
) )
elif self.master_opts['transport'] == 'raet':
expect = ''
self.assertEqual( self.assertEqual(
''.join(data), ''.join(data),
expect expect

4
tests/jenkins.py
View file

@ -848,9 +848,9 @@ def parse():
parser.add_option( parser.add_option(
'--test-transport', '--test-transport',
default='zeromq', default='zeromq',
choices=('zeromq', 'raet', 'tcp'), choices=('zeromq', 'tcp'),
help=('Select which transport to run the integration tests with, ' help=('Select which transport to run the integration tests with, '
'zeromq, raet, or tcp. Default: %default') 'zeromq or tcp. Default: %default')
) )
parser.add_option( parser.add_option(
'--test-without-coverage', '--test-without-coverage',

7
tests/minionswarm.py
View file

@ -166,7 +166,6 @@ class Swarm(object):
''' '''
def __init__(self, opts): def __init__(self, opts):
self.opts = opts self.opts = opts
self.raet_port = 4550
# If given a temp_dir, use it for temporary files # If given a temp_dir, use it for temporary files
if opts['temp_dir']: if opts['temp_dir']:
@ -329,12 +328,6 @@ class MinionSwarm(Swarm):
shutil.copy(minion_pem, minion_pkidir) shutil.copy(minion_pem, minion_pkidir)
shutil.copy(minion_pub, minion_pkidir) shutil.copy(minion_pub, minion_pkidir)
data['pki_dir'] = minion_pkidir data['pki_dir'] = minion_pkidir
elif self.opts['transport'] == 'raet':
data['transport'] = 'raet'
data['sock_dir'] = os.path.join(dpath, 'sock')
data['raet_port'] = self.raet_port
data['pki_dir'] = os.path.join(dpath, 'pki')
self.raet_port += 1
elif self.opts['transport'] == 'tcp': elif self.opts['transport'] == 'tcp':
data['transport'] = 'tcp' data['transport'] = 'tcp'

73
tests/perf/fire_fake.py
View file

@ -1,73 +0,0 @@
# -*- encoding: utf-8 -**
from __future__ import absolute_import, print_function
# Import system libs
import sys
import datetime
# Import salt libs
import salt.config
import salt.client.raet
try:
opts = salt.config.master_config('/etc/salt/master')
except OSError:
print('Could not open master config. Do you need to be root?')
sys.exit(1)
class SwarmController(object):
'''
A controlling class for instantiating and controlling worker procs
'''
def __init__(self, opts):
self.opts = opts
self.client = salt.client.raet.LocalClient(mopts=opts)
self.total_complete = 0
self.run_time = 90 # The number of seconds to run for
self.reqs_sec = 5000 # The number of requests / second to shoot for
self.granularity = 200 # Re-calibrate once for this many runs
self.period_sleep = 0.01 # The number of seconds to initially sleep between pubs
self.ramp_sleep = 0.001 # The number of seconds to ramp up up or down by per calibration
self.start_time = None # The timestamp for the initiation of the test run
def run(self):
'''
Run the sequence in a loop
'''
last_check = 0
self.start_time = datetime.datetime.now()
goal = self.reqs_sec * self.run_time
while True:
self.fire_it()
last_check += 1
if last_check > self.granularity:
self.calibrate()
last_check = 0
if self.total_complete > goal:
print('Test complete')
break
def fire_it(self):
'''
Send the pub!
'''
self.client.pub('silver', 'test.ping')
self.total_complete += 1
def calibrate(self):
'''
Re-calibrate the speed
'''
elapsed_time = datetime.datetime.now() - self.start_time
#remaining_time = self.run_time - elapsed_time
#remaining_requests = (self.reqs_sec * self.run_time) - self.total_complete
# Figure out what the reqs/sec has been up to this point and then adjust up or down
runtime_reqs_sec = self.total_complete / elapsed_time.total_seconds()
print('Recalibrating. Current reqs/sec: {0}'.format(runtime_reqs_sec))
return
controller = SwarmController(opts)
controller.run()

4
tests/runtests.py
View file

@ -261,9 +261,9 @@ class SaltTestsuiteParser(SaltCoverageTestingParser):
self.add_option( self.add_option(
'--transport', '--transport',
default='zeromq', default='zeromq',
choices=('zeromq', 'raet', 'tcp'), choices=('zeromq', 'tcp'),
help=('Select which transport to run the integration tests with, ' help=('Select which transport to run the integration tests with, '
'zeromq, raet, or tcp. Default: %default') 'zeromq or tcp. Default: %default')
) )
self.add_option( self.add_option(
'--interactive', '--interactive',

1
tests/support/mixins.py
View file

@ -103,7 +103,6 @@ class AdaptedConfigurationTestCaseMixin(object):
os.path.join(rdict['pki_dir'], 'minions_rejected'), os.path.join(rdict['pki_dir'], 'minions_rejected'),
os.path.join(rdict['pki_dir'], 'minions_denied'), os.path.join(rdict['pki_dir'], 'minions_denied'),
os.path.join(rdict['cachedir'], 'jobs'), os.path.join(rdict['cachedir'], 'jobs'),
os.path.join(rdict['cachedir'], 'raet'),
os.path.join(rdict['cachedir'], 'tokens'), os.path.join(rdict['cachedir'], 'tokens'),
os.path.join(rdict['root_dir'], 'cache', 'tokens'), os.path.join(rdict['root_dir'], 'cache', 'tokens'),
os.path.join(rdict['pki_dir'], 'accepted'), os.path.join(rdict['pki_dir'], 'accepted'),

6
tests/unit/modules/test_event.py
View file

@ -47,12 +47,6 @@ class EventTestCase(TestCase, LoaderModuleMockMixin):
preload = {'id': 'id', 'tag': 'tag', 'data': 'data', preload = {'id': 'id', 'tag': 'tag', 'data': 'data',
'tok': 'salt', 'cmd': '_minion_event'} 'tok': 'salt', 'cmd': '_minion_event'}
with patch.dict(event.__opts__, {'transport': 'raet',
'local': False}):
with patch.object(salt_transport_channel_factory, 'send',
return_value=None):
self.assertTrue(event.fire_master('data', 'tag'))
with patch.dict(event.__opts__, {'transport': 'A', with patch.dict(event.__opts__, {'transport': 'A',
'master_uri': 'localhost', 'master_uri': 'localhost',
'local': False}): 'local': False}):

75
tests/unit/modules/test_raet_publish.py
View file

@ -1,75 +0,0 @@
# -*- coding: utf-8 -*-
'''
:codeauthor: Jayesh Kariya <jayeshk@saltstack.com>
'''
# Import Python libs
from __future__ import absolute_import, print_function, unicode_literals
# Import Salt Testing Libs
from tests.support.mixins import LoaderModuleMockMixin
from tests.support.unit import skipIf, TestCase
from tests.support.mock import (
NO_MOCK,
NO_MOCK_REASON,
MagicMock,
patch)
# Import Salt Libs
import salt.modules.raet_publish as raet_publish
import salt.transport.client
from salt.exceptions import SaltReqTimeoutError
@skipIf(NO_MOCK, NO_MOCK_REASON)
class RaetPublishTestCase(TestCase, LoaderModuleMockMixin):
'''
Test cases for salt.modules.raet_publish
'''
def setup_loader_modules(self):
return {raet_publish: {}}
def test_publish(self):
'''
Test for publish a command from the minion out to other minions.
'''
with patch.object(raet_publish, '_publish', return_value='A'):
self.assertEqual(raet_publish.publish('tgt', 'fun'), 'A')
def test_full_data(self):
'''
Test for return the full data about the publication,
this is invoked in the same way as the publish function
'''
with patch.object(raet_publish, '_publish', return_value='A'):
self.assertEqual(raet_publish.full_data('tgt', 'fun'), 'A')
def test_runner(self):
'''
Test for execute a runner on the master and return
the data from the runner function
'''
with patch.dict(raet_publish.__opts__, {'id': 'id'}):
with patch.object(salt.transport.client.ReqChannel, 'factory', MagicMock()):
self.assertTrue(raet_publish.runner('fun'))
class MockFactory(object):
'''
Mock factory class
'''
load = ''
def send(self, load):
'''
mock send method
'''
self.load = load
raise SaltReqTimeoutError(load)
def close(self):
pass
with patch.dict(raet_publish.__opts__, {'id': 'id'}):
with patch.object(salt.transport.client.ReqChannel, 'factory',
MagicMock(return_value=MockFactory())):
self.assertEqual(raet_publish.runner(1), "'1' runner publish timed out")