diff --git a/debian/control b/debian/control index 36b9ae074bc..402a57f52df 100644 --- a/debian/control +++ b/debian/control @@ -160,6 +160,28 @@ Description: public cloud VM management system controlled profile and mapping system. +Package: salt-api +Architecture: all +Depends: ${python:Depends}, + ${misc:Depends}, + ${shlibs:Depends}, + python, + salt-master +Recommends: python-cherrypy3 +Description: Generic, modular network access system + a modular interface on top of Salt that can provide a variety of entry points + into a running Salt system. It can start and manage multiple interfaces + allowing a REST API to coexist with XMLRPC or even a Websocket API. + . + The Salt API system is used to expose the fundamental aspects of Salt control + to external sources. salt-api acts as the bridge between Salt itself and + REST, Websockets, etc. + . + Documentation is available on Read the Docs: + . + http://salt-api.readthedocs.org/ + + Package: salt-doc Architecture: all Section: doc diff --git a/debian/salt-api.install b/debian/salt-api.install new file mode 100644 index 00000000000..47a7f2406cb --- /dev/null +++ b/debian/salt-api.install @@ -0,0 +1 @@ +scripts/salt-api /usr/bin diff --git a/debian/salt-api.manpages b/debian/salt-api.manpages new file mode 100644 index 00000000000..cf17c09f016 --- /dev/null +++ b/debian/salt-api.manpages @@ -0,0 +1,2 @@ +doc/man/salt-api.1 +doc/man/salt-api.7 diff --git a/doc/conf.py b/doc/conf.py index bb24b1a6b5e..ced8dd0799c 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -24,10 +24,14 @@ class Mock(object): pass def __call__(self, *args, **kwargs): - return Mock() + ret = Mock() + # If mocked function is used as a decorator, expose decorated function. + if args and callable(args[0]): + functools.update_wrapper(ret, args[0]) + return ret @classmethod - def __getattr__(self, name): + def __getattr__(cls, name): if name in ('__file__', '__path__'): return '/dev/null' else: @@ -48,7 +52,8 @@ MOCK_MODULES = [ 'yaml.nodes', 'yaml.scanner', 'zmq', - # salt.cloud + + # third-party libs for cloud modules 'libcloud', 'libcloud.compute', 'libcloud.compute.base', @@ -60,6 +65,27 @@ MOCK_MODULES = [ 'libcloud.loadbalancer.providers', 'libcloud.common', 'libcloud.common.google', + + # third-party libs for netapi modules + 'cherrypy', + 'cherrypy.lib', + 'cherrypy.process', + 'cherrypy.wsgiserver', + 'cherrypy.wsgiserver.ssl_builtin', + + 'tornado', + 'tornado.concurrent', + 'tornado.gen', + 'tornado.httpserver', + 'tornado.ioloop', + 'tornado.web', + 'tornado.websocket', + + 'ws4py', + 'ws4py.server', + 'ws4py.server.cherrypyserver', + 'ws4py.websocket', + # modules, renderers, states, returners, et al 'django', 'libvirt', @@ -139,6 +165,7 @@ extensions = [ 'sphinx.ext.autosummary', 'sphinx.ext.extlinks', 'sphinx.ext.intersphinx', + 'sphinxcontrib.httpdomain', 'youtube', 'saltautodoc', # Must be AFTER autodoc 'shorturls', diff --git a/doc/index.rst b/doc/index.rst index bf6e823e132..716b96685ea 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -225,9 +225,13 @@ Salt is many splendid things. Controlling devices and machines unable to run a salt-minion. :ref:`Python API interface ` - Use Salt programmatically from scripts and programs easily and + Use Salt locally from scripts and programs easily and simply via ``import salt``. +:ref:`External API interfaces ` + Expose a Salt API such as REST, XMPP, WebSockets, and more using netapi + modules. See the :ref:`full list of netapi modules `. + :doc:`Automatic Updates and Frozen Binary Deployments ` Use a frozen install to make deployments easier (Even on Windows!). Or take advantage of automatic updates to keep minions running the latest diff --git a/doc/man/salt-api.1 b/doc/man/salt-api.1 new file mode 100644 index 00000000000..ea619ff6f9e --- /dev/null +++ b/doc/man/salt-api.1 @@ -0,0 +1,70 @@ +.\" Man page generated from reStructuredText. +. +.TH "SALT-API" "1" "April 05, 2014" "0.8.3" "salt-api" +.SH NAME +salt-api \- salt-api +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.sp +Start interfaces used to remotely connect to the salt master +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +salt\-api +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DESCRIPTION +.sp +The Salt API system manages network api connectors for the Salt Master +.SH OPTIONS +.INDENT 0.0 +.TP +.B \-h, \-\-help +Print a usage message briefly summarizing these command\-line options. +.UNINDENT +.INDENT 0.0 +.TP +.B \-C CONFIG, \-\-config=CONFIG +Specify an alternative location for the salt master configuration file. +.UNINDENT +.SH SEE ALSO +.sp +\fIsalt\-api(7)\fP +\fIsalt(7)\fP +\fIsalt\-master(1)\fP +.SH AUTHOR +Thomas S. Hatch and many others, please see the Authors file +.SH COPYRIGHT +2012, Thomas S. Hatch +.\" Generated by docutils manpage writer. +. diff --git a/doc/man/salt-api.7 b/doc/man/salt-api.7 new file mode 100644 index 00000000000..721e46993b1 --- /dev/null +++ b/doc/man/salt-api.7 @@ -0,0 +1,2237 @@ +.\" Man page generated from reStructuredText. +. +.TH "SALT-API" "7" "April 05, 2014" "0.8.3" "salt-api" +.SH NAME +salt-api \- salt-api Documentation +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.sp +\fBsalt\-api\fP is a modular interface on top of \fI\%Salt\fP that can provide a +variety of entry points into a running Salt system. It can start and manage +multiple interfaces allowing a REST API to coexist with XMLRPC or even a +Websocket API. +.SH GETTING STARTED +.INDENT 0.0 +.IP 1. 3 +Install \fBsalt\-api\fP on the same machine as your Salt master. +.IP 2. 3 +Edit your Salt master config file for all required options for each +\fBnetapi\fP module you wish to run. +.IP 3. 3 +Install any required additional libraries or software for each \fBnetapi\fP +module you wish to run. +.IP 4. 3 +Run \fBsalt\-api\fP which will then start all configured \fBnetapi\fP +modules. +.UNINDENT +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +Each \fBnetapi\fP module will have differing configuration requirements and +differing required software libraries. +.sp +Exactly like the various module types in Salt (\fIexecution modules\fP, +\fIrenderer modules\fP, \fIreturner modules\fP, etc.), \fInetapi +modules\fP in \fBsalt\-api\fP will \fInot\fP be loaded into memory or started +if all requirements are not met. +.UNINDENT +.UNINDENT +.SH INSTALLATION QUICKSTART +.SS salt\-api Quickstart +.sp +\fBsalt\-api\fP manages \fInetapi modules\fP which are modules that +(usually) bind to a port and start a service. Each netapi module will have +specific requirements for third\-party libraries and configuration (which goes +in the master config file). Read the documentation for each netapi module to +determine what is needed. +.sp +For example, the \fBrest_cherrypy\fP +netapi module requires that CherryPy be installed and that a \fBrest_cherrypy\fP +section be added to the master config that specifies which port to listen on. +.SS Installation +.SS PyPI +.sp +\fI\%https://pypi.python.org/pypi/salt\-api\fP +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pip install salt\-api +.ft P +.fi +.UNINDENT +.UNINDENT +.SS RHEL, Fedora, CentOS +.sp +RPMs are available in the Fedora repositories and EPEL: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +yum install salt\-api +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Ubuntu +.sp +PPA packages available for Ubuntu on LaunchPad: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +sudo add\-apt\-repository ppa:saltstack/salt +sudo apt\-get update +sudo apt\-get install salt\-api +.ft P +.fi +.UNINDENT +.UNINDENT +.SS openSUSE, SLES +.sp +RPMs are available via the OBS: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +zypper install salt\-api +.ft P +.fi +.UNINDENT +.UNINDENT +.SH NETAPI MODULES +.sp +The core functionality for \fBsalt\-api\fP lies in pluggable \fBnetapi\fP +modules that adhere to the simple interface of binding to a port and starting a +service. \fBsalt\-api\fP can manage one or many services concurrently. +.SS Full list of \fBnetapi\fP modules +.SS Full list of netapi modules +.SS rest_cherrypy +.SS A REST API for Salt +.INDENT 0.0 +.TP +.B depends +.INDENT 7.0 +.IP \(bu 2 +CherryPy Python module +.UNINDENT +.TP +.B configuration +All authentication is done through Salt\(aqs \fI\%external auth\fP system. Be sure that it is enabled and the user you are +authenticating as has permissions for all the functions you will be +running. +.sp +Example production configuration block; add to the Salt master config file: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +rest_cherrypy: + port: 8000 + ssl_crt: /etc/pki/tls/certs/localhost.crt + ssl_key: /etc/pki/tls/certs/localhost.key +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +The REST interface strongly recommends a secure HTTPS connection since Salt +authentication credentials will be sent over the wire. If you don\(aqt already +have a certificate and don\(aqt wish to buy one, you can generate a +self\-signed certificate using the +\fI\%create_self_signed_cert()\fP function in Salt (note +the dependencies for this module): +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% salt\-call tls.create_self_signed_cert +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +All available configuration options are detailed below. These settings +configure the CherryPy HTTP server and do not apply when using an external +server such as Apache or Nginx. +.INDENT 7.0 +.TP +.B port +\fBRequired\fP +.sp +The port for the webserver to listen on. +.TP +.B host +\fB0.0.0.0\fP +The socket interface for the HTTP server to listen on. +.sp +New in version 0.8.2. + +.TP +.B debug +\fBFalse\fP +Starts the web server in development mode. It will reload itself when +the underlying code is changed and will output more debugging info. +.TP +.B ssl_crt +The path to a SSL certificate. (See below) +.TP +.B ssl_key +The path to the private key for your SSL certificate. (See below) +.TP +.B disable_ssl +A flag to disable SSL. Warning: your Salt authentication credentials +will be sent in the clear! +.sp +New in version 0.8.3. + +.TP +.B thread_pool +\fB100\fP +The number of worker threads to start up in the pool. +.sp +Changed in version 0.8.4: Previous versions defaulted to a pool of \fB10\fP + +.TP +.B socket_queue_size +\fB30\fP +Specify the maximum number of HTTP connections to queue. +.sp +Changed in version 0.8.4: Previous versions defaulted to \fB5\fP connections. + +.TP +.B max_request_body_size +\fB1048576\fP +Changed in version 0.8.4: Previous versions defaulted to \fB104857600\fP for the size of the +request body + +.TP +.B collect_stats +False +Collect and report statistics about the CherryPy server +.sp +New in version 0.8.4. + +.sp +Reports are available via the \fBStats\fP URL. +.TP +.B static +A filesystem path to static HTML/JavaScript/CSS/image assets. +.TP +.B static_path +\fB/static\fP +The URL prefix to use when serving static assets out of the directory +specified in the \fBstatic\fP setting. +.sp +New in version 0.8.2. + +.TP +.B app +A filesystem path to an HTML file that will be served as a static file. +This is useful for bootstrapping a single\-page JavaScript app. +.sp +New in version 0.8.2. + +.TP +.B app_path +\fB/app\fP +The URL prefix to use for serving the HTML file specified in the \fBapp\fP +setting. This should be a simple name containing no slashes. +.sp +Any path information after the specified path is ignored; this is +useful for apps that utilize the HTML5 history API. +.sp +New in version 0.8.2. + +.TP +.B root_prefix +\fB/\fP +A URL path to the main entry point for the application. This is useful +for serving multiple applications from the same URL. +.sp +New in version 0.8.4. + +.UNINDENT +.UNINDENT +.SS Authentication +.sp +Authentication is performed by passing a session token with each request. The +token may be sent either via a custom header named \fIX\-Auth\-Token\fP +or sent inside a cookie. (The result is the same but browsers and some HTTP +clients handle cookies automatically and transparently so it is a convenience.) +.sp +Token are generated via the \fBLogin\fP URL. +.sp +\fBSEE ALSO:\fP +.INDENT 0.0 +.INDENT 3.5 +You can bypass the session handling via the \fBRun\fP URL. +.UNINDENT +.UNINDENT +.SS Usage +.sp +You access a running Salt master via this module by sending HTTP requests to +the URLs detailed below. +.INDENT 0.0 +.INDENT 3.5 +.IP "Content negotiation" +.sp +This REST interface is flexible in what data formats it will accept as well +as what formats it will return (e.g., JSON, YAML, x\-www\-form\-urlencoded). +.INDENT 0.0 +.IP \(bu 2 +Specify the format of data you are sending in a request by including the +\fIContent\-Type\fP header. +.IP \(bu 2 +Specify your desired output format for the response with the +\fIAccept\fP header. +.UNINDENT +.UNINDENT +.UNINDENT +.sp +This REST interface expects data sent in \fI\%POST\fP and +\fI\%PUT\fP requests to be in the format of a list of lowstate +dictionaries. This allows you to specify multiple commands in a single request. +.INDENT 0.0 +.TP +.B lowstate +A dictionary containing various keys that instruct Salt which command +to run, where that command lives, any parameters for that command, any +authentication credentials, what returner to use, etc. +.sp +Salt uses the lowstate data format internally in many places to pass +command data between functions. Salt also uses lowstate for the +\fI\%LocalClient()\fP Python API interface. +.UNINDENT +.sp +For example (in JSON format): +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[{ + \(aqclient\(aq: \(aqlocal\(aq, + \(aqtgt\(aq: \(aq*\(aq, + \(aqfun\(aq: \(aqtest.fib\(aq, + \(aqarg\(aq: [\(aq10\(aq], +}] +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.IP "x\-www\-form\-urlencoded" +.sp +This REST interface accepts data in the x\-www\-form\-urlencoded format. This +is the format used by HTML forms, the default format used by +\fBcurl\fP, the default format used by many JavaScript AJAX libraries +(such as jQuery), etc. This format will be converted to the +\fIlowstate\fP format as best as possible with the caveats below. It is +always preferable to format data in the lowstate format directly in a more +capable format such as JSON or YAML. +.INDENT 0.0 +.IP \(bu 2 +Only a single command may be sent in this format per HTTP request. +.IP \(bu 2 +Multiple \fBarg\fP params will be sent as a single list of params. +.sp +Note, some popular frameworks and languages (notably jQuery, PHP, and +Ruby on Rails) will automatically append empty brackets onto repeated +parameters. E.g., arg=one, arg=two will be sent as arg[]=one, arg[]=two. +Again, it is preferable to send lowstate via JSON or YAML directly by +specifying the \fIContent\-Type\fP header in the request. +.UNINDENT +.UNINDENT +.UNINDENT +.SS URL reference +.sp +The main entry point is the \fBroot URL (/)\fP and all +functionality is available at that URL. The other URLs are largely convenience +URLs that wrap that main entry point with shorthand or specialized +functionality. +.SS Deployment +.sp +The \fBrest_cherrypy\fP netapi module is a standard Python WSGI app. It can be +deployed one of two ways. +.SS \fBsalt\-api\fP using the CherryPy server +.sp +The default configuration is to run this module using \fBsalt\-api\fP to +start the Python\-based CherryPy server. This server is lightweight, +multi\-threaded, encrypted with SSL, and should be considered production\-ready. +.SS Using a WSGI\-compliant web server +.sp +This module may be deplayed on any WSGI\-compliant server such as Apache with +mod_wsgi or Nginx with FastCGI, to name just two (there are many). +.sp +Note, external WSGI servers handle URLs, paths, and SSL certs directly. The +\fBrest_cherrypy\fP configuration options are ignored and the \fBsalt\-api\fP daemon +does not need to be running at all. Remember Salt authentication credentials +are sent in the clear unless SSL is being enforced! +.sp +An example Apache virtual host configuration: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C + + ServerName example.com + ServerAlias *.example.com + + ServerAdmin webmaster@example.com + + LogLevel warn + ErrorLog /var/www/example.com/logs/error.log + CustomLog /var/www/example.com/logs/access.log combined + + DocumentRoot /var/www/example.com/htdocs + + WSGIScriptAlias / /path/to/saltapi/netapi/rest_cherrypy/wsgi.py + +.ft P +.fi +.UNINDENT +.UNINDENT +.SS REST URI Reference +.INDENT 0.0 +.TP +.B class saltapi.netapi.rest_cherrypy.app.LowDataAdapter +The primary entry point to the REST API. All functionality is available +through this URL. The other available URLs provide convenience wrappers +around this URL. +.INDENT 7.0 +.TP +.B GET() +.INDENT 7.0 +.TP +.B GET / +An explanation of the API with links of where to go next. +.sp +\fBExample request\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-i localhost:8000 +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +GET / HTTP/1.1 +Host: localhost:8000 +Accept: application/json +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBExample response\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +HTTP/1.1 200 OK +Content\-Type: application/json +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B Status 200 +success +.TP +.B Status 401 +authentication required +.TP +.B Status 406 +requested Content\-Type not available +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B class saltapi.netapi.rest_cherrypy.app.Login(*args, **kwargs) +All interactions with this REST API must be authenticated. Authentication +is performed through Salt\(aqs eauth system. You must set the eauth backend +and allowed users by editing the \fI\%external_auth\fP section in +your master config. +.sp +Authentication credentials are passed to the REST API via a session id in +one of two ways: +.sp +If the request is initiated from a browser it must pass a session id via a +cookie and that session must be valid and active. +.sp +If the request is initiated programmatically, the request must contain a +\fIX\-Auth\-Token\fP header with valid and active session id. +.INDENT 7.0 +.TP +.B GET() +Present the login interface +.INDENT 7.0 +.TP +.B GET /login +An explanation of how to log in. +.sp +\fBExample request\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-i localhost:8000/login +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +GET /login HTTP/1.1 +Host: localhost:8000 +Accept: text/html +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBExample response\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +HTTP/1.1 200 OK +Content\-Type: text/html +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B Status 401 +authentication required +.TP +.B Status 406 +requested Content\-Type not available +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B POST(**kwargs) +Authenticate against Salt\(aqs eauth system +.sp +Changed in version 0.8.0: No longer returns a 302 redirect on success. + +.sp +Changed in version 0.8.1: Returns 401 on authentication failure + +.INDENT 7.0 +.TP +.B POST /login +\fBExample request\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-si localhost:8000/login \e + \-H "Accept: application/json" \e + \-d username=\(aqsaltuser\(aq \e + \-d password=\(aqsaltpass\(aq \e + \-d eauth=\(aqpam\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +POST / HTTP/1.1 +Host: localhost:8000 +Content\-Length: 42 +Content\-Type: application/x\-www\-form\-urlencoded +Accept: application/json + +username=saltuser&password=saltpass&eauth=pam +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBExample response\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +HTTP/1.1 200 OK +Content\-Type: application/json +Content\-Length: 206 +X\-Auth\-Token: 6d1b722e +Set\-Cookie: session_id=6d1b722e; expires=Sat, 17 Nov 2012 03:23:52 GMT; Path=/ + +{"return": { + "token": "6d1b722e", + "start": 1363805943.776223, + "expire": 1363849143.776224, + "user": "saltuser", + "eauth": "pam", + "perms": [ + "grains.*", + "status.*", + "sys.*", + "test.*" + ] +}} +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B Form eauth +the eauth backend configured in your master config +.TP +.B Form username +username +.TP +.B Form password +password +.TP +.B Status 200 +success +.TP +.B Status 401 +could not authenticate using provided credentials +.TP +.B Status 406 +requested Content\-Type not available +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B class saltapi.netapi.rest_cherrypy.app.Logout +.INDENT 7.0 +.TP +.B POST() +Destroy the currently active session and expire the session cookie +.sp +New in version 0.8.0. + +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B class saltapi.netapi.rest_cherrypy.app.Minions +.INDENT 7.0 +.TP +.B GET(mid=None) +A convenience URL for getting lists of minions or getting minion +details +.INDENT 7.0 +.TP +.B GET /minions/(mid) +Get grains, modules, functions, and inline function documentation +for all minions or a single minion +.sp +\fBExample request\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-i localhost:8000/minions/ms\-3 +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +GET /minions/ms\-3 HTTP/1.1 +Host: localhost:8000 +Accept: application/x\-yaml +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBExample response\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +HTTP/1.1 200 OK +Content\-Length: 129005 +Content\-Type: application/x\-yaml + +return: +\- ms\-3: + grains.items: + ... +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B Parameters +\fBmid\fP \-\- (optional) a minion id +.TP +.B Status 200 +success +.TP +.B Status 401 +authentication required +.TP +.B Status 406 +requested Content\-Type not available +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B POST(**kwargs) +Start an execution command and immediately return the job id +.INDENT 7.0 +.TP +.B POST /minions +You must pass low\-data in the request body either from an HTML form +or as JSON or YAML. The \fBclient\fP option is pre\-set to +\fBlocal_async\fP\&. +.sp +\fBExample request\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-sSi localhost:8000/minions \e + \-H "Accept: application/x\-yaml" \e + \-d tgt=\(aq*\(aq \e + \-d fun=\(aqstatus.diskusage\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +POST /minions HTTP/1.1 +Host: localhost:8000 +Accept: application/x\-yaml +Content\-Length: 26 +Content\-Type: application/x\-www\-form\-urlencoded + +tgt=*&fun=status.diskusage +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBExample response\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +HTTP/1.1 202 Accepted +Content\-Length: 86 +Content\-Type: application/x\-yaml + +return: +\- jid: \(aq20130603122505459265\(aq + minions: [ms\-4, ms\-3, ms\-2, ms\-1, ms\-0] +_links: + jobs: + \- href: /jobs/20130603122505459265 +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B Form lowstate +lowstate data for the +\fBLocalClient\fP; the \fBclient\fP parameter will +be set to \fBlocal_async\fP +.sp +Lowstate may be supplied in any supported format by specifying the +\fIContent\-Type\fP header in the request. Supported formats +are listed in the \fIAlternates\fP response header. +.TP +.B Status 202 +success +.TP +.B Status 401 +authentication required +.TP +.B Status 406 +requested \fIContent\-Type\fP not available +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B class saltapi.netapi.rest_cherrypy.app.Jobs +.INDENT 7.0 +.TP +.B GET(jid=None) +A convenience URL for getting lists of previously run jobs or getting +the return from a single job +.INDENT 7.0 +.TP +.B GET /jobs/(jid) +Get grains, modules, functions, and inline function documentation +for all minions or a single minion +.sp +\fBExample request\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-i localhost:8000/jobs +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +GET /jobs HTTP/1.1 +Host: localhost:8000 +Accept: application/x\-yaml +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBExample response\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +HTTP/1.1 200 OK +Content\-Length: 165 +Content\-Type: application/x\-yaml + +return: +\- \(aq20121130104633606931\(aq: + Arguments: + \- \(aq3\(aq + Function: test.fib + Start Time: 2012, Nov 30 10:46:33.606931 + Target: jerry + Target\-type: glob +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBExample request\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-i localhost:8000/jobs/20121130104633606931 +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +GET /jobs/20121130104633606931 HTTP/1.1 +Host: localhost:8000 +Accept: application/x\-yaml +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBExample response\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +HTTP/1.1 200 OK +Content\-Length: 73 +Content\-Type: application/x\-yaml + +info: +\- Arguments: + \- \(aq3\(aq + Function: test.fib + Minions: + \- jerry + Start Time: 2012, Nov 30 10:46:33.606931 + Target: \(aq*\(aq + Target\-type: glob + User: saltdev + jid: \(aq20121130104633606931\(aq +return: +\- jerry: + \- \- 0 + \- 1 + \- 1 + \- 2 + \- 6.9141387939453125e\-06 +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B Parameters +\fBmid\fP \-\- (optional) a minion id +.TP +.B Status 200 +success +.TP +.B Status 401 +authentication required +.TP +.B Status 406 +requested Content\-Type not available +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B class saltapi.netapi.rest_cherrypy.app.Run +.INDENT 7.0 +.TP +.B POST(**kwargs) +Run commands bypassing the normal session handling +.sp +New in version 0.8.0. + +.INDENT 7.0 +.TP +.B POST /run +This entry point is primarily for "one\-off" commands. Each request +must pass full Salt authentication credentials. Otherwise this URL +is identical to the root (\fB/\fP) execution URL. +.sp +\fBExample request\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-sS localhost:8000/run \e + \-H \(aqAccept: application/x\-yaml\(aq \e + \-d client=\(aqlocal\(aq \e + \-d tgt=\(aq*\(aq \e + \-d fun=\(aqtest.ping\(aq \e + \-d username=\(aqsaltdev\(aq \e + \-d password=\(aqsaltdev\(aq \e + \-d eauth=\(aqpam\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +POST /run HTTP/1.1 +Host: localhost:8000 +Accept: application/x\-yaml +Content\-Length: 75 +Content\-Type: application/x\-www\-form\-urlencoded + +client=local&tgt=*&fun=test.ping&username=saltdev&password=saltdev&eauth=pam +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBExample response\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +HTTP/1.1 200 OK +Content\-Length: 73 +Content\-Type: application/x\-yaml + +return: +\- ms\-0: true + ms\-1: true + ms\-2: true + ms\-3: true + ms\-4: true +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B Form lowstate +A list of \fIlowstate\fP data appropriate for the +\fI\%client\fP specified client interface. Full +external authentication credentials must be included. +.TP +.B Status 200 +success +.TP +.B Status 401 +authentication failed +.TP +.B Status 406 +requested Content\-Type not available +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B class saltapi.netapi.rest_cherrypy.app.Events +The event bus on the Salt master exposes a large variety of things, notably +when executions are started on the master and also when minions ultimately +return their results. This URL provides a real\-time window into a running +Salt infrastructure. +.INDENT 7.0 +.TP +.B GET(token=None) +Return an HTTP stream of the Salt master event bus; this stream is +formatted per the Server Sent Events (SSE) spec +.sp +New in version 0.8.3. + +.sp +Browser clients currently lack Cross\-origin resource sharing (CORS) +support for the \fBEventSource()\fP API. Cross\-domain requests from a +browser may instead pass the \fIX\-Auth\-Token\fP value as an URL +parameter: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-NsS localhost:8000/events/6d1b722e +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B GET /events +\fBExample request\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-NsS localhost:8000/events +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +GET /events HTTP/1.1 +Host: localhost:8000 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBExample response\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +HTTP/1.1 200 OK +Connection: keep\-alive +Cache\-Control: no\-cache +Content\-Type: text/event\-stream;charset=utf\-8 + +retry: 400 +data: {\(aqtag\(aq: \(aq\(aq, \(aqdata\(aq: {\(aqminions\(aq: [\(aqms\-4\(aq, \(aqms\-3\(aq, \(aqms\-2\(aq, \(aqms\-1\(aq, \(aqms\-0\(aq]}} + +data: {\(aqtag\(aq: \(aq20130802115730568475\(aq, \(aqdata\(aq: {\(aqjid\(aq: \(aq20130802115730568475\(aq, \(aqreturn\(aq: True, \(aqretcode\(aq: 0, \(aqsuccess\(aq: True, \(aqcmd\(aq: \(aq_return\(aq, \(aqfun\(aq: \(aqtest.ping\(aq, \(aqid\(aq: \(aqms\-1\(aq}} +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.sp +The event stream can be easily consumed via JavaScript: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +# Note, you must be authenticated! +var source = new EventSource(\(aq/events\(aq); +source.onopen = function() { console.debug(\(aqopening\(aq) }; +source.onerror = function(e) { console.debug(\(aqerror!\(aq, e) }; +source.onmessage = function(e) { console.debug(e.data) }; +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +It is also possible to consume the stream via the shell. +.sp +Records are separated by blank lines; the \fBdata:\fP and \fBtag:\fP +prefixes will need to be removed manually before attempting to +unserialize the JSON. +.sp +curl\(aqs \fB\-N\fP flag turns off input buffering which is required to +process the stream incrementally. +.sp +Here is a basic example of printing each event as it comes in: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-NsS localhost:8000/events |\e + while IFS= read \-r line ; do + echo $line + done +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Here is an example of using awk to filter events based on tag: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-NsS localhost:8000/events |\e + awk \(aq + BEGIN { RS=""; FS="\en" } + $1 ~ /^tag: salt\e/job\e/[0\-9]+\e/new$/ { print $0 } + \(aq +tag: salt/job/20140112010149808995/new +data: {"tag": "salt/job/20140112010149808995/new", "data": {"tgt_type": "glob", "jid": "20140112010149808995", "tgt": "jerry", "_stamp": "2014\-01\-12_01:01:49.809617", "user": "shouse", "arg": [], "fun": "test.ping", "minions": ["jerry"]}} +tag: 20140112010149808995 +data: {"tag": "20140112010149808995", "data": {"fun_args": [], "jid": "20140112010149808995", "return": true, "retcode": 0, "success": true, "cmd": "_return", "_stamp": "2014\-01\-12_01:01:49.819316", "fun": "test.ping", "id": "jerry"}} +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B Status 200 +success +.TP +.B Status 401 +could not authenticate using provided credentials +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B class saltapi.netapi.rest_cherrypy.app.Webhook +A generic web hook entry point that fires an event on Salt\(aqs event bus +.sp +External services can POST data to this URL to trigger an event in Salt. +For example, Jenkins\-CI or Travis\-CI, or GitHub web hooks. +.sp +This entry point does not require authentication. The event data is taken +from the request body. +.sp +\fBNOTE:\fP +.INDENT 7.0 +.INDENT 3.5 +Be mindful of security +.sp +Salt\(aqs Reactor can run any code. If you write a Reactor SLS that +responds to a hook event be sure to validate that the event came from a +trusted source and contains valid data! Pass a secret key and use SSL. +.sp +This is a generic interface and securing it is up to you! +.UNINDENT +.UNINDENT +.sp +The event tag is prefixed with \fBsalt/netapi/hook\fP and the URL path is +appended to the end. For example, a \fBPOST\fP request sent to +\fB/hook/mycompany/myapp/mydata\fP will produce a Salt event with the tag +\fBsalt/netapi/hook/mycompany/myapp/mydata\fP\&. See the \fI\%Salt Reactor\fP documentation for how to react to events with various tags. +.sp +The following is an example \fB\&.travis.yml\fP file to send notifications to +Salt of successful test runs: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +language: python +script: python \-m unittest tests +after_success: + \- \(aqcurl \-sS http://saltapi\-url.example.com:8000/hook/travis/build/success \-d branch="${TRAVIS_BRANCH}" \-d commit="${TRAVIS_COMMIT}"\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B POST(*args, **kwargs) +Fire an event in Salt with a custom event tag and data +.sp +New in version 0.8.4. + +.INDENT 7.0 +.TP +.B POST /hook +\fBExample request\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-sS localhost:8000/hook \-d foo=\(aqFoo!\(aq \-d bar=\(aqBar!\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +POST /hook HTTP/1.1 +Host: localhost:8000 +Content\-Length: 16 +Content\-Type: application/x\-www\-form\-urlencoded + +foo=Foo&bar=Bar! +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBExample response\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +HTTP/1.1 200 OK +Content\-Length: 14 +Content\-Type: application/json + +{"success": true} +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.sp +As a practical example, an internal continuous\-integration build +server could send an HTTP POST request to the URL +\fBhttp://localhost:8000/hook/mycompany/build/success\fP which contains +the result of a build and the SHA of the version that was built as +JSON. That would then produce the following event in Salt that could be +used to kick off a deployment via Salt\(aqs Reactor: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +Event fired at Fri Feb 14 17:40:11 2014 +************************* +Tag: salt/netapi/hook/mycompany/build/success +Data: +{\(aq_stamp\(aq: \(aq2014\-02\-14_17:40:11.440996\(aq, + \(aqheaders\(aq: { + \(aqX\-My\-Secret\-Key\(aq: \(aqF0fAgoQjIT@W\(aq, + \(aqContent\-Length\(aq: \(aq37\(aq, + \(aqContent\-Type\(aq: \(aqapplication/json\(aq, + \(aqHost\(aq: \(aqlocalhost:8000\(aq, + \(aqRemote\-Addr\(aq: \(aq127.0.0.1\(aq}, + \(aqpost\(aq: {\(aqrevision\(aq: \(aqaa22a3c4b2e7\(aq, \(aqresult\(aq: True}} +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Salt\(aqs Reactor could listen for the event: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +reactor: + \- \(aqsalt/netapi/hook/mycompany/build/*\(aq: + \- /srv/reactor/react_ci_builds.sls +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +And finally deploy the new build: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +{% set secret_key = data.get(\(aqheaders\(aq, {}).get(\(aqX\-My\-Secret\-Key\(aq) %} +{% set build = data.get(\(aqpost\(aq, {}) %} + +{% if secret_key == \(aqF0fAgoQjIT@W\(aq and build.result == True %} +deploy_my_app: + cmd.state.sls: + \- tgt: \(aqapplication*\(aq + \- arg: + \- myapp.deploy + \- \(aqpillar={revision: {{ revision }}}\(aq +{% endif %} +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B Status 200 +success +.TP +.B Status 406 +requested Content\-Type not available +.TP +.B Status 413 +request body is too large +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B class saltapi.netapi.rest_cherrypy.app.Stats +Expose statistics on the running CherryPy server +.INDENT 7.0 +.TP +.B GET() +Return a dump of statistics collected from the CherryPy server +.INDENT 7.0 +.TP +.B Status 200 +success +.TP +.B Status 406 +requested Content\-Type not available +.UNINDENT +.UNINDENT +.UNINDENT +.SS rest_wsgi +.SS A minimalist REST API for Salt +.sp +This \fBrest_wsgi\fP module provides a no\-frills REST interface to a running Salt +master. There are no dependencies. +.sp +Please read this introductory section in entirety before deploying this module. +.INDENT 0.0 +.TP +.B configuration +All authentication is done through Salt\(aqs \fI\%external auth\fP system. Be sure that it is enabled and the user you are +authenticating as has permissions for all the functions you will be +running. +.sp +The configuration options for this module resides in the Salt master config +file. All available options are detailed below. +.INDENT 7.0 +.TP +.B port +\fBRequired\fP +.sp +The port for the webserver to listen on. +.UNINDENT +.sp +Example configuration: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +rest_wsgi: + port: 8001 +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.sp +This API is not very "RESTful"; please note the following: +.INDENT 0.0 +.IP \(bu 2 +All requests must be sent to the root URL (\fB/\fP). +.IP \(bu 2 +All requests must be sent as a POST request with JSON content in the request +body. +.IP \(bu 2 +All responses are in JSON. +.UNINDENT +.sp +\fBSEE ALSO:\fP +.INDENT 0.0 +.INDENT 3.5 +\fBrest_cherrypy\fP +.sp +The \fBrest_cherrypy\fP module is +more full\-featured, production\-ready, and has builtin security features. +.UNINDENT +.UNINDENT +.SS Deployment +.sp +The \fBrest_wsgi\fP netapi module is a standard Python WSGI app. It can be +deployed one of two ways. +.SS \fBsalt\-api\fP using a development\-only server +.sp +If run directly via salt\-api it uses the \fI\%wsgiref.simple_server()\fP that ships +in the Python standard library. This is a single\-threaded server that is +intended for testing and development. This server does \fBnot\fP use encryption; +please note that raw Salt authentication credentials must be sent with every +HTTP request. +.sp +\fBRunning this module via salt\-api is not recommended for most use!\fP +.SS Using a WSGI\-compliant web server +.sp +This module may be run via any WSGI\-compliant production server such as Apache +with mod_wsgi or Nginx with FastCGI. +.sp +It is highly recommended that this app be used with a server that supports +HTTPS encryption since raw Salt authentication credentials must be sent with +every request. Any apps that access Salt through this interface will need to +manually manage authentication credentials (either username and password or a +Salt token). Tread carefully. +.SS Usage examples +.INDENT 0.0 +.TP +.B POST / +\fBExample request\fP for a basic \fBtest.ping\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-sS \-i \e + \-H \(aqContent\-Type: application/json\(aq \e + \-d \(aq[{"eauth":"pam","username":"saltdev","password":"saltdev","client":"local","tgt":"*","fun":"test.ping"}]\(aq localhost:8001 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBExample response\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +HTTP/1.0 200 OK +Content\-Length: 89 +Content\-Type: application/json + +{"return": [{"ms\-\-4": true, "ms\-\-3": true, "ms\-\-2": true, "ms\-\-1": true, "ms\-\-0": true}]} +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBExample request\fP for an asyncronous \fBtest.ping\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-sS \-i \e + \-H \(aqContent\-Type: application/json\(aq \e + \-d \(aq[{"eauth":"pam","username":"saltdev","password":"saltdev","client":"local_async","tgt":"*","fun":"test.ping"}]\(aq localhost:8001 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBExample response\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +HTTP/1.0 200 OK +Content\-Length: 103 +Content\-Type: application/json + +{"return": [{"jid": "20130412192112593739", "minions": ["ms\-\-4", "ms\-\-3", "ms\-\-2", "ms\-\-1", "ms\-\-0"]}]} +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBExample request\fP for looking up a job ID: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-sS \-i \e + \-H \(aqContent\-Type: application/json\(aq \e + \-d \(aq[{"eauth":"pam","username":"saltdev","password":"saltdev","client":"runner","fun":"jobs.lookup_jid","jid":"20130412192112593739"}]\(aq localhost:8001 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBExample response\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +HTTP/1.0 200 OK +Content\-Length: 89 +Content\-Type: application/json + +{"return": [{"ms\-\-4": true, "ms\-\-3": true, "ms\-\-2": true, "ms\-\-1": true, "ms\-\-0": true}]} +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B form lowstate +A list of \fIlowstate\fP data appropriate for the +\fI\%client\fP interface you are calling. +.TP +.B status 200 +success +.TP +.B status 401 +authentication required +.UNINDENT +.SS \fBnetapi\fP developer reference +.SS Introduction to netapi modules +.sp +netapi modules generally bind to a port and start a service. They are +purposefully open\-ended. There could be multiple netapi modules that provide a +REST interface, a module that provides an XMPP interface, or Websockets, or +XMLRPC. +.sp +netapi modules are enabled by adding configuration to your master config file. +Check the docs for each module to see external requirements and configuration +settings. +.sp +Communication with Salt and Salt satellite projects is done by passing a list of +lowstate dictionaries to a client interface. A list of available client +interfaces is below. The lowstate dictionary items map to keyword arguments on +the client interface. +.sp +\fBSEE ALSO:\fP +.INDENT 0.0 +.INDENT 3.5 +\fI\%Python client API\fP +.UNINDENT +.UNINDENT +.SS Client interfaces +.INDENT 0.0 +.TP +.B class saltapi.APIClient(opts) +Provide a uniform method of accessing the various client interfaces in Salt +in the form of low\-data data structures. For example: +.sp +.nf +.ft C +>>> client = APIClient(__opts__) +>>> lowstate = {\(aqclient\(aq: \(aqlocal\(aq, \(aqtgt\(aq: \(aq*\(aq, \(aqfun\(aq: \(aqtest.ping\(aq, \(aqarg\(aq: \(aq\(aq} +>>> client.run(lowstate) +.ft P +.fi +.INDENT 7.0 +.TP +.B local(*args, **kwargs) +Run \fI\%execution modules\fP syncronously +.sp +Wraps \fI\%salt.client.LocalClient.cmd()\fP\&. +.INDENT 7.0 +.TP +.B Returns +Returns the result from the execution module +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B local_async(*args, **kwargs) +Run \fI\%execution modules\fP asyncronously +.sp +Wraps \fI\%salt.client.LocalClient.run_job()\fP\&. +.INDENT 7.0 +.TP +.B Returns +job ID +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B local_batch(*args, **kwargs) +Run \fI\%execution modules\fP against batches of minions +.sp +New in version 0.8.4. + +.sp +Wraps \fI\%salt.client.LocalClient.cmd_batch()\fP +.INDENT 7.0 +.TP +.B Returns +Returns the result from the exeuction module for each batch of +returns +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B runner(fun, **kwargs) +Run \fIrunner modules \fP +.sp +Wraps \fI\%salt.runner.RunnerClient.low()\fP\&. +.INDENT 7.0 +.TP +.B Returns +Returns the result from the runner module +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B wheel(fun, **kwargs) +Run \fI\%wheel modules\fP +.sp +Wraps \fI\%salt.wheel.WheelClient.master_call()\fP\&. +.INDENT 7.0 +.TP +.B Returns +Returns the result from the wheel module +.UNINDENT +.UNINDENT +.UNINDENT +.SS Writing netapi modules +.sp +\fBnetapi\fP modules, put simply, bind a port and start a service. +They are purposefully open\-ended and can be used to present a variety of +external interfaces to Salt, and even present multiple interfaces at once. +.sp +\fBSEE ALSO:\fP +.INDENT 0.0 +.INDENT 3.5 +\fIThe full list of netapi modules\fP +.UNINDENT +.UNINDENT +.SS Configuration +.sp +All \fBnetapi\fP configuration is done in the \fI\%Salt master +config\fP and takes a form similar to the following: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +rest_cherrypy: + port: 8000 + debug: True + ssl_crt: /etc/pki/tls/certs/localhost.crt + ssl_key: /etc/pki/tls/certs/localhost.key +.ft P +.fi +.UNINDENT +.UNINDENT +.SS The \fB__virtual__\fP function +.sp +Like all module types in Salt, \fBnetapi\fP modules go through +Salt\(aqs loader interface to determine if they should be loaded into memory and +then executed. +.sp +The \fB__virtual__\fP function in the module makes this determination and should +return \fBFalse\fP or a string that will serve as the name of the module. If the +module raises an \fBImportError\fP or any other errors, it will not be loaded. +.SS The \fBstart\fP function +.sp +The \fBstart()\fP function will be called for each \fBnetapi\fP +module that is loaded. This function should contain the server loop that +actually starts the service. This is started in a multiprocess. +.SS Inline documentation +.sp +As with the rest of Salt, it is a best\-practice to include liberal inline +documentation in the form of a module docstring and docstrings on any classes, +methods, and functions in your \fBnetapi\fP module. +.SS Loader “magic” methods +.sp +The loader makes the \fB__opts__\fP data structure available to any function in +a \fBnetapi\fP module. +.SH RELEASES +.SS Release notes +.SS salt\-api 0.5.0 +.sp +\fBsalt\-api\fP is gearing up for the initial public release with 0.5.0. +Although this release ships with working basic functionality it is awaiting the +authentication backend that will be introduced in Salt 0.10.4 before it can be +considered ready for testing at large. +.SS REST API +.sp +This release presents the flagship netapi module which provides a RESTful +interface to a running Salt system. It allows for viewing minions, runners, and +jobs as well as running execution modules and runners of a running Salt system +through a REST API that returns JSON. +.SS Participation +.sp +\fBsalt\-api\fP is just getting off the ground so feedback, questions, and +ideas are critical as we solidify how this project fits into the overall Salt +infrastructure management stack. Please get involved by \fI\%filing issues\fP on +GitHub, \fI\%discussing on the mailing list\fP, and chatting in \fB#salt\fP on +Freenode. +.SS salt\-api 0.6.0 +.sp +\fBsalt\-api\fP inches closer to prime\-time with 0.6.0. This release adds +the beginnings of a universal interface for accessing Salt components via the +tried and true method of passing low\-data to functions (a core component of +Salt\(aqs remote execution and state management). +.SS Low\-data interface +.sp +A new view accepts :\fI\%http:post\fP: requests at the root URL that accepts raw +low\-data as :\fI\%http:post\fP: data and passes that low\-data along to a client +interface in Salt. Currently only LocalClient and RunnerClient interfaces have +been implemented in Salt with more coming in the next Salt release. +.SS External authentication +.sp +Raw low\-data can contain authentication credentials that make use of Salt\(aqs new +\fI\%external_auth\fP system. +.sp +The following is a proof\-of\-concept of a working eauth call. (It bears +repeating this is a pre\-alpha release and this should not be used by anyone for +anything real.) +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-si localhost:8000 \e + \-d client=local \e + \-d tgt=\(aq*\(aq \e + \-d fun=\(aqtest.ping\(aq \e + \-d arg \e + \-d eauth=pam \e + \-d username=saltdev \e + \-d password=saltdev +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Participation +.sp +\fBsalt\-api\fP is just getting off the ground so feedback, questions, and +ideas are critical as we solidify how this project fits into the overall Salt +infrastructure management stack. Please get involved by \fI\%filing issues\fP on +GitHub, \fI\%discussing on the mailing list\fP, and chatting in \fB#salt\-devel\fP on +Freenode. +.SS salt\-api 0.7.0 +.sp +\fBsalt\-api\fP is ready for alpha\-testing in the real world. This release +solidifies how \fBsalt\-api\fP will communicate with the larger Salt +ecosystem. In addition authentication and encryption (via SSL) have been added. +.sp +The first netapi module was a proof of concept written in Flask. It was quite +useful to be able to quickly hammer out a URL structure and solidify on an +interface for programmatically calling out to Salt components. As of this +release that module has been deprecated and removed in favor of a netapi module +written in CherryPy. CherryPy affords tremendous flexibility when composing a +REST interface and will present a stable platform for building out a very +adaptable and featureful REST API—also we\(aqre using the excellent and fast +CherryPy webserver for securely serving the API. +.SS Low\-data interface +.sp +The last release introduced a proof\-of\-concept for how the various Salt +components will communicate with each other. This is done by passing a data +structure to a client interface. This release expands on that. There are +currently three client interfaces in Salt. +.sp +\fBSEE ALSO:\fP +.INDENT 0.0 +.INDENT 3.5 +\fInetapi\-introduction\fP +.UNINDENT +.UNINDENT +.SS Encryption and authentication +.sp +Encryption has been added via SSL. You can supply an existing certificate or +generate a self\-signed certificate through Salt\(aqs \fI\%tls\fP +module. +.sp +Authentication is performed through Salt\(aqs incredibly flexible \fI\%external +auth\fP system and is maintained when accessing the API via session +tokens. +.SS Participation +.sp +\fBsalt\-api\fP is just getting off the ground so feedback, questions, and +ideas are critical as we solidify how this project fits into the overall Salt +infrastructure management stack. Please get involved by \fI\%filing issues\fP on +GitHub, \fI\%discussing on the mailing list\fP, and chatting in \fB#salt\-devel\fP on +Freenode. +.SS salt\-api 0.7.5 +.sp +This release is a mostly a minor release to pave a better path for +\fBsalt\-ui\fP though there are some small feature additions and bugfixes. +.SS Changes +.INDENT 0.0 +.IP \(bu 2 +Convenience URLs \fB/minions\fP and \fB/jobs\fP have been added as well as a +async client wrapper. This starts a job and immediately returns the job ID, +allowing you to fetch the result of that job at a later time. +.IP \(bu 2 +The return format will now default to JSON if no specific format is +requested. +.IP \(bu 2 +A new setting \fBstatic\fP has been added that will serve any static media from +the directory specified. In addition if an \fBindex.html\fP file is found +in that directory and the \fBAccept\fP header in the request prefer HTML that +file will be served. +.IP \(bu 2 +All HTML, including the login form, has been removed from \fBsalt\-api\fP +and moved into the \fBsalt\-ui\fP project. +.IP \(bu 2 +Sessions now live as long as the Salt token. +.UNINDENT +.SS Participation +.sp +\fBsalt\-api\fP is just getting off the ground so feedback, questions, and +ideas are critical as we solidify how this project fits into the overall Salt +infrastructure management stack. Please get involved by \fI\%filing issues\fP on +GitHub, \fI\%discussing on the mailing list\fP, and chatting in \fB#salt\-devel\fP on +Freenode. +.SS salt\-api 0.8.0 +.sp +We are happy to announce the release of \fBsalt\-api\fP 0.8.0. +.sp +This release encompasses bugfixes and new features for the +\fBrest_cherrypy\fP netapi module that +provides a RESTful interface for a running Salt system. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +Requires Salt 0.13 +.UNINDENT +.UNINDENT +.SS Changes +.sp +In addition to the usual documentation improvements and bug fixes this release +introduces the following changes and additions. +.sp +Please note the backward incompatible change detailed below. +.SS RPM packaging +.sp +Thanks to Andrew Niemantsvedriet (\fI\%@kaptk2\fP) \fBsalt\-api\fP is now +available in Fedora package repositories as well as RHEL compatible systems via +EPEL. +.INDENT 0.0 +.IP \(bu 2 +\fI\%http://dl.fedoraproject.org/pub/epel/5/i386/repoview/salt\-api.html\fP +.IP \(bu 2 +\fI\%http://dl.fedoraproject.org/pub/epel/5/x86_64/repoview/salt\-api.html\fP +.IP \(bu 2 +\fI\%http://dl.fedoraproject.org/pub/epel/6/i386/repoview/salt\-api.html\fP +.IP \(bu 2 +\fI\%http://dl.fedoraproject.org/pub/epel/6/x86_64/repoview/salt\-api.html\fP +.UNINDENT +.sp +Thanks also to Clint Savage (\fI\%@herlo\fP) and Thomas Spura (\fI\%@tomspur\fP) for +helping with that process. +.SS Ubuntu PPA packaging +.sp +Thanks to Sean Channel (\fI\%@seanchannel\fP, pentabular) \fBsalt\-api\fP is +available as a PPA on the SaltStack LaunchPad team. +.sp +\fI\%https://launchpad.net/~saltstack/+archive/salt\fP +.SS Authentication information on login +.sp +\fBWARNING:\fP +.INDENT 0.0 +.INDENT 3.5 +Backward incompatible change +.sp +The \fB/login\fP URL no +longer responds with a 302 redirect for success. +.sp +Although this is behavior is common in the browser world it is not useful +from an API so we have changed it to return a 200 response in this release. +.sp +We take backward compatibility very seriously and we apologize for the +inconvenience. In this case we felt the previous behavior was limiting. +Changes such as this will be rare. +.UNINDENT +.UNINDENT +.sp +New in this release is displaying information about the current session and the +current user. For example: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-sS localhost:8000/login \e + \-H \(aqAccept: application/x\-yaml\(aq + \-d username=\(aqsaltdev\(aq + \-d password=\(aqsaltdev\(aq + \-d eauth=\(aqpam\(aq + +return: +\- eauth: pam + expire: 1365508324.359403 + perms: + \- \(aq@wheel\(aq + \- grains.* + \- state.* + \- status.* + \- sys.* + \- test.* + start: 1365465124.359402 + token: caa7aa2b9dbc4a8adb6d2e19c3e52be68995ef4b + user: saltdev +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Bypass session handling +.sp +A convenience URL has been added +(\fB/run\fP) to bypass the normal +session\-handling process. +.sp +The REST interface uses the concept of "lowstate" data to specify what function +should be executed in Salt (plus where that function is and any arguments to +the function). This is a thin wrapper around Salt\(aqs various "client" +interfaces, for example Salt\(aqs \fI\%LocalClient()\fP which can +accept authentication credentials directly. +.sp +Authentication with the REST API typically goes through the login URL and a +session is generated that is tied to a Salt external_auth token. That token is +then automatically added to the lowstate for subsequent requests that match the +current session. +.sp +It is sometimes useful to handle authentication or token management manually +from another program or script. For example: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +curl \-sS localhost:8000/run \e + \-d client=\(aqlocal\(aq \e + \-d tgt=\(aq*\(aq \e + \-d fun=\(aqtest.ping\(aq \e + \-d eauth=\(aqpam\(aq \e + \-d username=\(aqsaltdev\(aq \e + \-d password=\(aqsaltdev\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +It is a Bad Idea (TM) to do this unless you have a very good reason and a well +thought out security model. +.SS Logout +.sp +An URL has been added +(\fB/logout\fP) that will cause +the client\-side to expire the session cookie and the server\-side session to be +invalidated. +.SS Running the REST interface via any WSGI\-compliant server +.sp +The \fBrest_cherrypy\fP netapi module is +a regular WSGI application written using the CherryPy framework. It was written +with the intent of also running from any WSGI\-compliant server such as Apache +and mod_wsgi, Gunicorn, uWSGI, Nginx and FastCGI, etc. +.sp +The WSGI application entry point has been factored out into a stand\-alone file +in this release suitable for calling from an external server. +\fBsalt\-api\fP does not need to be running in this scenario. +.sp +For example, an Apache virtual host configuration: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C + + ServerName example.com + ServerAlias *.example.com + + ServerAdmin webmaster@example.com + + LogLevel warn + ErrorLog /var/www/example.com/logs/error.log + CustomLog /var/www/example.com/logs/access.log combined + + DocumentRoot /var/www/example.com/htdocs + + WSGIScriptAlias / /path/to/saltapi/netapi/rest_cherrypy/wsgi.py + +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Participation +.sp +Please get involved by \fI\%filing issues\fP on GitHub, \fI\%discussing on the mailing +list\fP, and chatting in \fB#salt\-devel\fP on Freenode. +.SS salt\-api 0.8.2 +.sp +\fBsalt\-api\fP 0.8.2 is largely a bugfix release that fixes a +compatibility issue with changes in Salt 0.15.9. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +Requires Salt 0.15.9 or greater +.UNINDENT +.UNINDENT +.sp +The following changes have been made to the \fBrest_cherrypy\fP netapi module that provides a RESTful +interface for a running Salt system: +.INDENT 0.0 +.IP \(bu 2 +Fixed issue #87 which caused the Salt master\(aqs PID file to be overwritten. +.IP \(bu 2 +Fixed an inconsistency with the return format for the \fB/minions\fP +convenience URL. +.sp +\fBWARNING:\fP +.INDENT 2.0 +.INDENT 3.5 +This is a backward incompatible change. +.UNINDENT +.UNINDENT +.IP \(bu 2 +Added a dedicated URL for serving an HTML app +.IP \(bu 2 +Added dedicated URL for serving static media +.UNINDENT +.SS salt\-api 0.8.3 +.sp +\fBsalt\-api\fP 0.8.3 is a small release largely concerning changes and +fixes to the \fBrest_cherrypy\fP netapi +module. +.sp +This release will likely be the final salt\-api release as a separate project. +The Salt team has begun the process of merging this project directly in to the +main Salt project. What this means for end users is only that there will be one +fewer package to install. Salt itself will ship with the current \fBnetapi\fP +modules and the API and configuration will remain otherwise unchanged. +.sp +The reasoning behind merging the two projects is simply to lower the barrier to +entry. Having a separate project was useful for experimentation and exploration +but there was no technical reason for the separation \-\- salt\-api uses the same +flexible module system that Salt uses and those modules will simply be moved +into Salt. +.sp +Going forward, Salt will ship with the same REST interface that salt\-api +currently provides. This will have the side benefit of not having to coordinate +incompatible Salt and salt\-api releases. +.SS \fBrest_cherrypy\fP changes +.sp +An HTTP stream of Salt\(aqs event bus has been added. This stream conforms to the +SSE (Server Sent Events) spec and is easily consumed via JavaScript clients. +This HTTP stream allows a real\-time window into a running Salt system. A client +watching the stream can see as soon as individual minions return data for a +job, authentication events, and any other events that go through the Salt +master. +.sp +A new configuration option to only allow access to whitelisted IP addresses. Of +course, IP addresses can be easily spoofed so this feature should be thought of +as a usability addition and not used for security purposes. +.sp +An option to disable SSL has been added. Previously SSL could only be disabled +while running the HTTP server with debugging options enabled. Now each item can +be enabled or disabled independently of the other. +.sp +In addition, there has been several bug fixes, packaging fixes, and minor code +simplification. +.SS salt\-api 0.8.4 +.sp +\fBsalt\-api\fP 0.8.4 sees a number of new features and feature +enhancements in the \fBrest_cherrypy\fP +netapi module. +.sp +Work to merge \fBsalt\-api\fP into the main Salt distribution continues and +it is likely to be included in Salt\(aqs Helium release. +.SS \fBrest_cherrypy\fP changes +.SS Web hooks +.sp +This release adds a \fBnew URL /hook\fP that allows salt\-api to serve as a +generic web hook interface for Salt. POST requests to the URL trigger events on +Salt\(aqs event bus. +.sp +External services like Amazon SNS, Travis CI, GitHub, etc can easily send +signals through Salt\(aqs Reactor. +.sp +The following HTTP call will trigger the following Salt event. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +% curl \-sS http://localhost:8000/hook/some/tag \e + \-d some=\(aqData!\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Event tag: \fBsalt/netapi/hook/some/tag\fP\&. Event data: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +{ + "_stamp": "2014\-04\-04T12:14:54.389614", + "post": { + "some": "Data!" + }, + "headers": { + "Content\-Type": "application/x\-www\-form\-urlencoded", + "Host": "localhost:8000", + "User\-Agent": "curl/7.32.0", + "Accept": "*/*", + "Content\-Length": "10", + "Remote\-Addr": "127.0.0.1" + } +} +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Batch mode +.sp +The \fBlocal_batch()\fP client exposes Salt\(aqs batch mode +for executing commands on incremental subsets of minions. +.SS Tests! +.sp +We have added the necessary framework for testing the rest_cherrypy module and +this release includes a number of both unit and integration tests. The suite +can be run with the following command: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +python \-m unittest discover \-v +.ft P +.fi +.UNINDENT +.UNINDENT +.SS CherryPy server stats and configuration +.sp +A number of settings have been added to better configure the performance of the +CherryPy web server. In addition, a \fBnew URL /stats\fP has been added to expose metrics on +the health of the CherryPy web server. +.SS Improvements for running with external WSGI servers +.sp +Running the \fBrest_cherrypy\fP module via a WSGI\-capable server such as Apache +or Nginx can be tricky since the user the server is running as must have +permission to access the running Salt system. This release eases some of those +restrictions by accessing Salt\(aqs key interface through the external auth +system. Read access to the Salt configuration is required for the user the +server is running as and everything else should go through external auth. +.SS More information in the jobs URLs +.sp +The output for the \fB/jobs/\fP has been augmented with more +information about the job such as which minions are expected to return for that +job. This same output will be added to the other salt\-api URLs in the next +release. +.SS Improvements to the Server Sent Events stream +.sp +Event tags have been added to \fBthe HTTP event stream\fP as SSE tags which allows JavaScript +or other consumers to more easily match on certain tags without having to +inspect the whole event. +.SH REFERENCE +.INDENT 0.0 +.IP \(bu 2 +\fIgenindex\fP +.IP \(bu 2 +\fImodindex\fP +.IP \(bu 2 +\fIsearch\fP +.IP \(bu 2 +\fIglossary\fP +.UNINDENT +.SH AUTHOR +Thomas S. Hatch and many others, please see the Authors file +.SH COPYRIGHT +2012, Thomas S. Hatch +.\" Generated by docutils manpage writer. +. diff --git a/doc/ref/cli/salt-api.rst b/doc/ref/cli/salt-api.rst new file mode 100644 index 00000000000..73ff77a9320 --- /dev/null +++ b/doc/ref/cli/salt-api.rst @@ -0,0 +1,37 @@ +============ +``salt-api`` +============ + +Start interfaces used to remotely connect to the salt master + +Synopsis +======== + +:: + + salt-api + +Description +=========== + +The Salt API system manages network api connectors for the Salt Master + +Options +======= + +.. program:: salt-api + +.. option:: -h, --help + + Print a usage message briefly summarizing these command-line options. + +.. option:: -C CONFIG, --config=CONFIG + + Specify an alternative location for the salt master configuration file. + +See also +======== + +:manpage:`salt-api(7)` +:manpage:`salt(7)` +:manpage:`salt-master(1)` diff --git a/doc/ref/netapi/all/index.rst b/doc/ref/netapi/all/index.rst new file mode 100644 index 00000000000..bb71b9ce5f8 --- /dev/null +++ b/doc/ref/netapi/all/index.rst @@ -0,0 +1,12 @@ +.. _all-netapi-modules: + +=========================== +Full list of netapi modules +=========================== + +.. toctree:: + :maxdepth: 2 + + salt.netapi.rest_cherrypy + salt.netapi.rest_tornado + salt.netapi.rest_wsgi diff --git a/doc/ref/netapi/all/saltapi.netapi.rest_cherrypy.rst b/doc/ref/netapi/all/saltapi.netapi.rest_cherrypy.rst new file mode 100644 index 00000000000..b8528fb5b24 --- /dev/null +++ b/doc/ref/netapi/all/saltapi.netapi.rest_cherrypy.rst @@ -0,0 +1,77 @@ +============= +rest_cherrypy +============= + +.. automodule:: salt.netapi.rest_cherrypy.app + +.. automodule:: salt.netapi.rest_cherrypy.wsgi + +.. ............................................................................ + +REST URI Reference +================== + +.. py:currentmodule:: salt.netapi.rest_cherrypy.app + +.. contents:: + :local: + +``/`` +----- + +.. autoclass:: LowDataAdapter + :members: GET, POST + +``/login`` +---------- + +.. autoclass:: Login + :members: GET, POST + +``/logout`` +----------- + +.. autoclass:: Logout + :members: POST + +``/minions`` +------------ + +.. autoclass:: Minions + :members: GET, POST + +``/jobs`` +--------- + +.. autoclass:: Jobs + :members: GET + +``/run`` +-------- + +.. autoclass:: Run + :members: POST + +``/events`` +----------- + +.. autoclass:: Events + :members: GET + +``/ws`` +------- + +.. autoclass:: WebsocketEndpoint + :members: GET + +``/hook`` +--------- + +.. autoclass:: Webhook + :members: POST + +``/stats`` +---------- + +.. autoclass:: Stats + :members: GET diff --git a/doc/ref/netapi/all/saltapi.netapi.rest_tornado.rst b/doc/ref/netapi/all/saltapi.netapi.rest_tornado.rst new file mode 100644 index 00000000000..5ee57bbae16 --- /dev/null +++ b/doc/ref/netapi/all/saltapi.netapi.rest_tornado.rst @@ -0,0 +1,7 @@ +============= +rest_tornado +============= + +.. automodule:: salt.netapi.rest_tornado.saltnado + +.. ............................................................................ diff --git a/doc/ref/netapi/all/saltapi.netapi.rest_wsgi.rst b/doc/ref/netapi/all/saltapi.netapi.rest_wsgi.rst new file mode 100644 index 00000000000..4536cd64134 --- /dev/null +++ b/doc/ref/netapi/all/saltapi.netapi.rest_wsgi.rst @@ -0,0 +1,7 @@ +========= +rest_wsgi +========= + +.. automodule:: salt.netapi.rest_wsgi + +.. py:currentmodule:: salt.netapi.rest_wsgi diff --git a/doc/topics/netapi/index.rst b/doc/topics/netapi/index.rst new file mode 100644 index 00000000000..feb4d52ff04 --- /dev/null +++ b/doc/topics/netapi/index.rst @@ -0,0 +1,36 @@ +.. _netapi-introduction: + +============================== +Introduction to netapi modules +============================== + +netapi modules provide API-centric access to Salt. Usually externally-facing +services such as REST or WebSockets, XMPP, XMLRPC, etc. + +In general netapi modules bind to a port and start a service. They are +purposefully open-ended. A single module can be configured to run as well as +multiple modules simultaneously. + +netapi modules are enabled by adding configuration to your Salt Master config +file and then starting the :command:`salt-api` daemon. Check the docs for each +module to see external requirements and configuration settings. + +Communication with Salt and Salt satellite projects is done using Salt's own +:ref:`Python API `. A list of available client interfaces is below. + +.. admonition:: salt-api + + Prior to Salt's Helium release, netapi modules lived in the separate sister + projected ``salt-api``. That project has been merged into the main Salt + project. + +Client interfaces +================= + +Salt's client interfaces expose executing functions by crafting a dictionary of +values that are mapped to function arguments. This allows calling functions +simply by creating a data structure. (And this is exactly how much of Salt's +own internals work!) + +.. autoclass:: salt.netapi.APIClient + :members: local, local_async, local_batch, runner, wheel diff --git a/doc/topics/netapi/writing.rst b/doc/topics/netapi/writing.rst new file mode 100644 index 00000000000..42a78a5480b --- /dev/null +++ b/doc/topics/netapi/writing.rst @@ -0,0 +1,54 @@ +====================== +Writing netapi modules +====================== + +:py:mod:`~salt.netapi` modules, put simply, bind a port and start a service. +They are purposefully open-ended and can be used to present a variety of +external interfaces to Salt, and even present multiple interfaces at once. + +.. seealso:: :ref:`The full list of netapi modules ` + +Configuration +============= + +All :py:mod:`~salt.netapi` configuration is done in the :ref:`Salt master +config ` and takes a form similar to the following: + +.. code-block:: yaml + + rest_cherrypy: + port: 8000 + debug: True + ssl_crt: /etc/pki/tls/certs/localhost.crt + ssl_key: /etc/pki/tls/certs/localhost.key + +The ``__virtual__`` function +============================ + +Like all module types in Salt, :py:mod:`~salt.netapi` modules go through +Salt's loader interface to determine if they should be loaded into memory and +then executed. + +The ``__virtual__`` function in the module makes this determination and should +return ``False`` or a string that will serve as the name of the module. If the +module raises an ``ImportError`` or any other errors, it will not be loaded. + +The ``start`` function +====================== + +The ``start()`` function will be called for each :py:mod:`~salt.netapi` +module that is loaded. This function should contain the server loop that +actually starts the service. This is started in a multiprocess. + +Inline documentation +==================== + +As with the rest of Salt, it is a best-practice to include liberal inline +documentation in the form of a module docstring and docstrings on any classes, +methods, and functions in your :py:mod:`~salt.netapi` module. + +Loader “magic” methods +====================== + +The loader makes the ``__opts__`` data structure available to any function in +a :py:mod:`~salt.netapi` module. diff --git a/doc/topics/releases/saltapi/0.5.0.rst b/doc/topics/releases/saltapi/0.5.0.rst new file mode 100644 index 00000000000..79db868aa85 --- /dev/null +++ b/doc/topics/releases/saltapi/0.5.0.rst @@ -0,0 +1,28 @@ +============== +salt-api 0.5.0 +============== + +:program:`salt-api` is gearing up for the initial public release with 0.5.0. +Although this release ships with working basic functionality it is awaiting the +authentication backend that will be introduced in Salt 0.10.4 before it can be +considered ready for testing at large. + +REST API +======== + +This release presents the flagship netapi module which provides a RESTful +interface to a running Salt system. It allows for viewing minions, runners, and +jobs as well as running execution modules and runners of a running Salt system +through a REST API that returns JSON. + +Participation +============= + +:program:`salt-api` is just getting off the ground so feedback, questions, and +ideas are critical as we solidify how this project fits into the overall Salt +infrastructure management stack. Please get involved by `filing issues`__ on +GitHub, `discussing on the mailing list`__, and chatting in ``#salt`` on +Freenode. + +.. __: https://github.com/saltstack/salt-api/issues +.. __: https://groups.google.com/forum/#!forum/salt-users diff --git a/doc/topics/releases/saltapi/0.6.0.rst b/doc/topics/releases/saltapi/0.6.0.rst new file mode 100644 index 00000000000..d2dc56d2f6b --- /dev/null +++ b/doc/topics/releases/saltapi/0.6.0.rst @@ -0,0 +1,50 @@ +============== +salt-api 0.6.0 +============== + +:program:`salt-api` inches closer to prime-time with 0.6.0. This release adds +the beginnings of a universal interface for accessing Salt components via the +tried and true method of passing low-data to functions (a core component of +Salt's remote execution and state management). + + +Low-data interface +================== + +A new view accepts :http:post: requests at the root URL that accepts raw +low-data as :http:post: data and passes that low-data along to a client +interface in Salt. Currently only LocalClient and RunnerClient interfaces have +been implemented in Salt with more coming in the next Salt release. + +External authentication +----------------------- + +Raw low-data can contain authentication credentials that make use of Salt's new +:conf_master:`external_auth` system. + +The following is a proof-of-concept of a working eauth call. (It bears +repeating this is a pre-alpha release and this should not be used by anyone for +anything real.) + +.. code-block:: bash + + % curl -si localhost:8000 \ + -d client=local \ + -d tgt='*' \ + -d fun='test.ping' \ + -d arg \ + -d eauth=pam \ + -d username=saltdev \ + -d password=saltdev + +Participation +============= + +:program:`salt-api` is just getting off the ground so feedback, questions, and +ideas are critical as we solidify how this project fits into the overall Salt +infrastructure management stack. Please get involved by `filing issues`__ on +GitHub, `discussing on the mailing list`__, and chatting in ``#salt-devel`` on +Freenode. + +.. __: https://github.com/saltstack/salt-api/issues +.. __: https://groups.google.com/forum/#!forum/salt-users diff --git a/doc/topics/releases/saltapi/0.7.0.rst b/doc/topics/releases/saltapi/0.7.0.rst new file mode 100644 index 00000000000..003c22fb9b1 --- /dev/null +++ b/doc/topics/releases/saltapi/0.7.0.rst @@ -0,0 +1,49 @@ +============== +salt-api 0.7.0 +============== + +:program:`salt-api` is ready for alpha-testing in the real world. This release +solidifies how :program:`salt-api` will communicate with the larger Salt +ecosystem. In addition authentication and encryption (via SSL) have been added. + +The first netapi module was a proof of concept written in Flask. It was quite +useful to be able to quickly hammer out a URL structure and solidify on an +interface for programmatically calling out to Salt components. As of this +release that module has been deprecated and removed in favor of a netapi module +written in CherryPy. CherryPy affords tremendous flexibility when composing a +REST interface and will present a stable platform for building out a very +adaptable and featureful REST API—also we're using the excellent and fast +CherryPy webserver for securely serving the API. + +Low-data interface +================== + +The last release introduced a proof-of-concept for how the various Salt +components will communicate with each other. This is done by passing a data +structure to a client interface. This release expands on that. There are +currently three client interfaces in Salt. + +.. seealso:: :ref:`netapi-introduction` + +Encryption and authentication +============================= + +Encryption has been added via SSL. You can supply an existing certificate or +generate a self-signed certificate through Salt's :py:mod:`~salt.modules.tls` +module. + +Authentication is performed through Salt's incredibly flexible :ref:`external +auth ` system and is maintained when accessing the API via session +tokens. + +Participation +============= + +:program:`salt-api` is just getting off the ground so feedback, questions, and +ideas are critical as we solidify how this project fits into the overall Salt +infrastructure management stack. Please get involved by `filing issues`__ on +GitHub, `discussing on the mailing list`__, and chatting in ``#salt-devel`` on +Freenode. + +.. __: https://github.com/saltstack/salt-api/issues +.. __: https://groups.google.com/forum/#!forum/salt-users diff --git a/doc/topics/releases/saltapi/0.7.5.rst b/doc/topics/releases/saltapi/0.7.5.rst new file mode 100644 index 00000000000..0345a8db4fd --- /dev/null +++ b/doc/topics/releases/saltapi/0.7.5.rst @@ -0,0 +1,35 @@ +============== +salt-api 0.7.5 +============== + +This release is a mostly a minor release to pave a better path for +:program:`salt-ui` though there are some small feature additions and bugfixes. + +Changes +======= + +* Convenience URLs ``/minions`` and ``/jobs`` have been added as well as a + async client wrapper. This starts a job and immediately returns the job ID, + allowing you to fetch the result of that job at a later time. +* The return format will now default to JSON if no specific format is + requested. +* A new setting ``static`` has been added that will serve any static media from + the directory specified. In addition if an :file:`index.html` file is found + in that directory and the ``Accept`` header in the request prefer HTML that + file will be served. +* All HTML, including the login form, has been removed from :program:`salt-api` + and moved into the :program:`salt-ui` project. +* Sessions now live as long as the Salt token. + +Participation +============= + +:program:`salt-api` is just getting off the ground so feedback, questions, and +ideas are critical as we solidify how this project fits into the overall Salt +infrastructure management stack. Please get involved by `filing issues`__ on +GitHub, `discussing on the mailing list`__, and chatting in ``#salt-devel`` on +Freenode. + +.. __: https://github.com/saltstack/salt-api/issues +.. __: https://groups.google.com/forum/#!forum/salt-users + diff --git a/doc/topics/releases/saltapi/0.8.0.rst b/doc/topics/releases/saltapi/0.8.0.rst new file mode 100644 index 00000000000..79ec6482d5e --- /dev/null +++ b/doc/topics/releases/saltapi/0.8.0.rst @@ -0,0 +1,166 @@ +============== +salt-api 0.8.0 +============== + +We are happy to announce the release of :program:`salt-api` 0.8.0. + +This release encompasses bugfixes and new features for the +:py:mod:`rest_cherrypy ` netapi module that +provides a RESTful interface for a running Salt system. + +.. note:: + + Requires Salt 0.13 + +Changes +======= + +In addition to the usual documentation improvements and bug fixes this release +introduces the following changes and additions. + +Please note the backward incompatible change detailed below. + +RPM packaging +------------- + +Thanks to Andrew Niemantsvedriet (`@kaptk2`_) :program:`salt-api` is now +available in Fedora package repositories as well as RHEL compatible systems via +EPEL. + +* http://dl.fedoraproject.org/pub/epel/5/i386/repoview/salt-api.html +* http://dl.fedoraproject.org/pub/epel/5/x86_64/repoview/salt-api.html +* http://dl.fedoraproject.org/pub/epel/6/i386/repoview/salt-api.html +* http://dl.fedoraproject.org/pub/epel/6/x86_64/repoview/salt-api.html + +Thanks also to Clint Savage (`@herlo`_) and Thomas Spura (`@tomspur`_) for +helping with that process. + +.. _`@kaptk2`: https://github.com/kaptk2 +.. _`@herlo`: https://github.com/herlo +.. _`@tomspur`: https://github.com/tomspur + +Ubuntu PPA packaging +-------------------- + +Thanks to Sean Channel (`@seanchannel`_, pentabular) :program:`salt-api` is +available as a PPA on the SaltStack LaunchPad team. + +https://launchpad.net/~saltstack/+archive/salt + +.. _`@seanchannel`: https://github.com/seanchannel + +Authentication information on login +----------------------------------- + +.. warning:: Backward incompatible change + + The :py:class:`/login ` URL no + longer responds with a 302 redirect for success. + + Although this is behavior is common in the browser world it is not useful + from an API so we have changed it to return a 200 response in this release. + + We take backward compatibility very seriously and we apologize for the + inconvenience. In this case we felt the previous behavior was limiting. + Changes such as this will be rare. + +New in this release is displaying information about the current session and the +current user. For example:: + + % curl -sS localhost:8000/login \ + -H 'Accept: application/x-yaml' + -d username='saltdev' + -d password='saltdev' + -d eauth='pam' + + return: + - eauth: pam + expire: 1365508324.359403 + perms: + - '@wheel' + - grains.* + - state.* + - status.* + - sys.* + - test.* + start: 1365465124.359402 + token: caa7aa2b9dbc4a8adb6d2e19c3e52be68995ef4b + user: saltdev + +Bypass session handling +----------------------- + +A convenience URL has been added +(:py:class:`/run `) to bypass the normal +session-handling process. + +The REST interface uses the concept of "lowstate" data to specify what function +should be executed in Salt (plus where that function is and any arguments to +the function). This is a thin wrapper around Salt's various "client" +interfaces, for example Salt's :ref:`LocalClient() ` which can +accept authentication credentials directly. + +Authentication with the REST API typically goes through the login URL and a +session is generated that is tied to a Salt external_auth token. That token is +then automatically added to the lowstate for subsequent requests that match the +current session. + +It is sometimes useful to handle authentication or token management manually +from another program or script. For example:: + + curl -sS localhost:8000/run \ + -d client='local' \ + -d tgt='*' \ + -d fun='test.ping' \ + -d eauth='pam' \ + -d username='saltdev' \ + -d password='saltdev' + +It is a Bad Idea (TM) to do this unless you have a very good reason and a well +thought out security model. + +Logout +------ + +An URL has been added +(:py:class:`/logout `) that will cause +the client-side to expire the session cookie and the server-side session to be +invalidated. + +Running the REST interface via any WSGI-compliant server +-------------------------------------------------------- + +The :py:mod:`rest_cherrypy ` netapi module is +a regular WSGI application written using the CherryPy framework. It was written +with the intent of also running from any WSGI-compliant server such as Apache +and mod_wsgi, Gunicorn, uWSGI, Nginx and FastCGI, etc. + +The WSGI application entry point has been factored out into a stand-alone file +in this release suitable for calling from an external server. +:program:`salt-api` does not need to be running in this scenario. + +For example, an Apache virtual host configuration:: + + + ServerName example.com + ServerAlias *.example.com + + ServerAdmin webmaster@example.com + + LogLevel warn + ErrorLog /var/www/example.com/logs/error.log + CustomLog /var/www/example.com/logs/access.log combined + + DocumentRoot /var/www/example.com/htdocs + + WSGIScriptAlias / /path/to/salt/netapi/rest_cherrypy/wsgi.py + + +Participation +============= + +Please get involved by `filing issues`__ on GitHub, `discussing on the mailing +list`__, and chatting in ``#salt-devel`` on Freenode. + +.. __: https://github.com/saltstack/salt-api/issues +.. __: https://groups.google.com/forum/#!forum/salt-users diff --git a/doc/topics/releases/saltapi/0.8.2.rst b/doc/topics/releases/saltapi/0.8.2.rst new file mode 100644 index 00000000000..183f4480601 --- /dev/null +++ b/doc/topics/releases/saltapi/0.8.2.rst @@ -0,0 +1,25 @@ +============== +salt-api 0.8.2 +============== + +:program:`salt-api` 0.8.2 is largely a bugfix release that fixes a +compatibility issue with changes in Salt 0.15.9. + +.. note:: + + Requires Salt 0.15.9 or greater + +The following changes have been made to the :py:mod:`rest_cherrypy +` netapi module that provides a RESTful +interface for a running Salt system: + +* Fixed issue #87 which caused the Salt master's PID file to be overwritten. +* Fixed an inconsistency with the return format for the ``/minions`` + convenience URL. + + .. warning:: + + This is a backward incompatible change. + +* Added a dedicated URL for serving an HTML app +* Added dedicated URL for serving static media diff --git a/doc/topics/releases/saltapi/0.8.3.rst b/doc/topics/releases/saltapi/0.8.3.rst new file mode 100644 index 00000000000..5935ec72926 --- /dev/null +++ b/doc/topics/releases/saltapi/0.8.3.rst @@ -0,0 +1,44 @@ +============== +salt-api 0.8.3 +============== + +:program:`salt-api` 0.8.3 is a small release largely concerning changes and +fixes to the :py:mod:`rest_cherrypy ` netapi +module. + +This release will likely be the final salt-api release as a separate project. +The Salt team has begun the process of merging this project directly in to the +main Salt project. What this means for end users is only that there will be one +fewer package to install. Salt itself will ship with the current ``netapi`` +modules and the API and configuration will remain otherwise unchanged. + +The reasoning behind merging the two projects is simply to lower the barrier to +entry. Having a separate project was useful for experimentation and exploration +but there was no technical reason for the separation -- salt-api uses the same +flexible module system that Salt uses and those modules will simply be moved +into Salt. + +Going forward, Salt will ship with the same REST interface that salt-api +currently provides. This will have the side benefit of not having to coordinate +incompatible Salt and salt-api releases. + +:py:mod:`rest_cherrypy ` changes +================================================================== + +An HTTP stream of Salt's event bus has been added. This stream conforms to the +SSE (Server Sent Events) spec and is easily consumed via JavaScript clients. +This HTTP stream allows a real-time window into a running Salt system. A client +watching the stream can see as soon as individual minions return data for a +job, authentication events, and any other events that go through the Salt +master. + +A new configuration option to only allow access to whitelisted IP addresses. Of +course, IP addresses can be easily spoofed so this feature should be thought of +as a usability addition and not used for security purposes. + +An option to disable SSL has been added. Previously SSL could only be disabled +while running the HTTP server with debugging options enabled. Now each item can +be enabled or disabled independently of the other. + +In addition, there has been several bug fixes, packaging fixes, and minor code +simplification. diff --git a/doc/topics/releases/saltapi/0.8.4.rst b/doc/topics/releases/saltapi/0.8.4.rst new file mode 100644 index 00000000000..65f8d374199 --- /dev/null +++ b/doc/topics/releases/saltapi/0.8.4.rst @@ -0,0 +1,102 @@ +============== +salt-api 0.8.4 +============== + +:program:`salt-api` 0.8.4 sees a number of new features and feature +enhancements in the :py:mod:`rest_cherrypy ` +netapi module. + +Work to merge :program:`salt-api` into the main Salt distribution continues and +it is likely to be included in Salt's Helium release. + +:py:mod:`rest_cherrypy ` changes +================================================================== + +Web hooks +--------- + +This release adds a :py:class:`new URL /hook +` that allows salt-api to serve as a +generic web hook interface for Salt. POST requests to the URL trigger events on +Salt's event bus. + +External services like Amazon SNS, Travis CI, GitHub, etc can easily send +signals through Salt's Reactor. + +The following HTTP call will trigger the following Salt event. + +.. code-block:: bash + + % curl -sS http://localhost:8000/hook/some/tag \ + -d some='Data!' + +Event tag: ``salt/netapi/hook/some/tag``. Event data: + +.. code-block:: json + + { + "_stamp": "2014-04-04T12:14:54.389614", + "post": { + "some": "Data!" + }, + "headers": { + "Content-Type": "application/x-www-form-urlencoded", + "Host": "localhost:8000", + "User-Agent": "curl/7.32.0", + "Accept": "*/*", + "Content-Length": "10", + "Remote-Addr": "127.0.0.1" + } + } + +Batch mode +---------- + +The :py:meth:`~salt.APIClient.local_batch` client exposes Salt's batch mode +for executing commands on incremental subsets of minions. + +Tests! +------ + +We have added the necessary framework for testing the rest_cherrypy module and +this release includes a number of both unit and integration tests. The suite +can be run with the following command: + +.. code-block:: bash + + python -m unittest discover -v + +CherryPy server stats and configuration +--------------------------------------- + +A number of settings have been added to better configure the performance of the +CherryPy web server. In addition, a :py:class:`new URL /stats +` has been added to expose metrics on +the health of the CherryPy web server. + +Improvements for running with external WSGI servers +--------------------------------------------------- + +Running the ``rest_cherrypy`` module via a WSGI-capable server such as Apache +or Nginx can be tricky since the user the server is running as must have +permission to access the running Salt system. This release eases some of those +restrictions by accessing Salt's key interface through the external auth +system. Read access to the Salt configuration is required for the user the +server is running as and everything else should go through external auth. + +More information in the jobs URLs +--------------------------------- + +The output for the :py:class:`/jobs/ URLs +` has been augmented with more +information about the job such as which minions are expected to return for that +job. This same output will be added to the other salt-api URLs in the next +release. + +Improvements to the Server Sent Events stream +--------------------------------------------- + +Event tags have been added to :py:class:`the HTTP event stream +` as SSE tags which allows JavaScript +or other consumers to more easily match on certain tags without having to +inspect the whole event. diff --git a/doc/topics/releases/saltapi/index.rst b/doc/topics/releases/saltapi/index.rst new file mode 100644 index 00000000000..af5a39404e3 --- /dev/null +++ b/doc/topics/releases/saltapi/index.rst @@ -0,0 +1,9 @@ +============= +Release notes +============= + +.. releasestree:: + :maxdepth: 1 + :glob: + + * diff --git a/opt_requirements.txt b/opt_requirements.txt index ef19fc9d65f..17188574d97 100644 --- a/opt_requirements.txt +++ b/opt_requirements.txt @@ -3,3 +3,4 @@ timelib yappi >= 0.8.2 --allow-unverified python-novaclient > 2.17.0 python-gnupg +cherrypy>=3.2.2 diff --git a/pkg/arch/salt-api_PKGBUILD b/pkg/arch/salt-api_PKGBUILD new file mode 100644 index 00000000000..282a1e17fd0 --- /dev/null +++ b/pkg/arch/salt-api_PKGBUILD @@ -0,0 +1,32 @@ +# Maintainer: Christer Edwards + +pkgname=salt-api +pkgver=0.8.0 +pkgrel=1 +pkgdesc="Salt API is used to expose the fundamental aspects of Salt control to external sources." +arch=(any) +url="https://github.com/saltstack/salt-api" +license=("APACHE") +depends=('python2' + 'salt') + +backup=() + +makedepends=() +optdepends=() +options=() +conflicts=() + +source=("http://pypi.python.org/packages/source/s/${pkgname}/${pkgname}-${pkgver}.tar.gz" + salt-api.service) + +md5sums=('e9239a7184ced5d426696735456ee829' + '37f667db44f63fb5dd7b81acf736b0db') + +package() { + cd ${srcdir}/${pkgname}-${pkgver} + python2 setup.py install --root=${pkgdir}/ --optimize=1 + + install -Dm644 ${srcdir}/salt-api.service ${pkgdir}/usr/lib/systemd/system/salt-api.service + +} diff --git a/pkg/arch/salt-api_PKGBUILD-git b/pkg/arch/salt-api_PKGBUILD-git new file mode 100644 index 00000000000..65b0d8635e4 --- /dev/null +++ b/pkg/arch/salt-api_PKGBUILD-git @@ -0,0 +1,59 @@ +# Maintainer: Christer Edwards +pkgname=salt-api-git +_gitname=salt-api +pkgver=0.0.0 +pkgrel=1 +pkgdesc="Salt API is used to expose the fundamental aspects of Salt control to external sources." +arch=('i686' 'x86_64') +url="https://github.com/saltstack/salt-api" +license=("APACHE") +depends=('python2' + 'salt') +backup=() +makedepends=('git') +optdepends=() +options=() +conflicts=('salt-api') +provides=('salt-api') + +# makepkg 4.1 knows about git and will pull main branch +source=("git://github.com/saltstack/salt-api.git") + +# makepkg knows it's a git repo because the url starts with 'git' +# it then knows to checkout the branch 'pacman41' upon cloning, expediting versioning. +# branch="develop" +# source=("git://github.com/saltstack/salt-api.git#branch=$branch") + +# makepkg also knows about tags +#tags="v0.8.0" +#source=("git://github.com/saltstack/salt-api.git#tag=$tag") + +# because the sources are not static, skip checksums +md5sums=('SKIP') + +pkgver() { + cd "$srcdir/$_gitname" + echo $(git describe --always | sed 's/-/./g') + # for git, if the repo has no tags, comment out the above and uncomment the next line: + #echo "0.$(git rev-list --count $branch).$(git describe --always)" + # This will give you a count of the total commits and the hash of the commit you are on. + # Useful if you're making a repository with git packages so that they can have sequential + # version numbers. (Else a pacman -Syu may not update the package) +} + +#build() { +# cd "${srcdir}/${_gitname}" +# python2 setup.py build + # no need to build setup.py install will do this +#} + +package() { + cd "${srcdir}/${_gitname}" + export USE_SETUPTOOLS=true + python2 setup.py install --root=${pkgdir}/ --optimize=1 + + install -Dm644 ${srcdir}/salt-api/pkg/salt-api.service ${pkgdir}/usr/lib/systemd/system/salt-api.service + + # remove vcs leftovers + find "$pkgdir" -type d -name .git -exec rm -r '{}' + +} diff --git a/pkg/rpm/salt-api b/pkg/rpm/salt-api new file mode 100644 index 00000000000..a077e4e9f53 --- /dev/null +++ b/pkg/rpm/salt-api @@ -0,0 +1,153 @@ +#!/bin/sh +# +# Salt API +################################### + +# LSB header + +### BEGIN INIT INFO +# Provides: salt-api +# Required-Start: $local_fs $remote_fs $network $named $time +# Should-Start: $time ypbind smtp +# Required-Stop: $local_fs $remote_fs $network $named $time +# Should-Stop: ypbind smtp +# Default-Start: 3 5 +# Default-Stop: 0 1 2 6 +# Short-Description: Salt API control daemon +# Description: This is a daemon that controls the Salt API. +### END INIT INFO + + +# chkconfig header + +# chkconfig: 345 99 99 +# description: This is a daemon that controls the Salt API. +# +# processname: /usr/bin/salt-api + + +if [ -f /etc/default/salt ]; then + . /etc/default/salt +else + SALTAPI=/usr/bin/salt-api + PYTHON=/usr/bin/python +fi + +# Sanity checks. +[ -x $SALTAPI ] || exit 0 + +DEBIAN_VERSION=/etc/debian_version +SUSE_RELEASE=/etc/SuSE-release +# Source function library. +if [ -f $DEBIAN_VERSION ]; then + break +elif [ -f $SUSE_RELEASE -a -r /etc/rc.status ]; then + . /etc/rc.status +else + . /etc/rc.d/init.d/functions +fi + +SERVICE=salt-api +PROCESS=salt-api +CONFIG_ARGS="-d" +PID_FILE="/var/run/salt-api.pid" + +RETVAL=0 + +start() { + echo -n $"Starting salt-api daemon: " + if [ -f $SUSE_RELEASE ]; then + startproc -f -p /var/run/$SERVICE.pid $SALTAPI $CONFIG_ARGS + rc_status -v + elif [ -e $DEBIAN_VERSION ]; then + if [ -f $LOCKFILE ]; then + echo -n "already started, lock file found" + RETVAL=1 + elif $PYTHON $SALTAPI; then + echo -n "OK" + RETVAL=0 + fi + else + if status $PROCESS &> /dev/null; then + failure "Already running." + RETVAL=1 + else + daemon --pidfile=$PID_FILE --check $SERVICE $SALTAPI $CONFIG_ARGS + RETVAL=0 + fi + fi + RETVAL=$? + echo + return $RETVAL +} + +stop() { + echo -n $"Stopping salt-api daemon: " + if [ -f $SUSE_RELEASE ]; then + killproc -TERM $SALTAPI + rc_status -v + elif [ -f $DEBIAN_VERSION ]; then + # Added this since Debian's start-stop-daemon doesn't support spawned processes + if ps -ef | grep "$PYTHON $SALTAPI" | grep -v grep | awk '{print $2}' | xargs kill &> /dev/null; then + echo -n "OK" + RETVAL=0 + else + echo -n "Daemon is not started" + RETVAL=1 + fi + else + if [ -f $PID_FILE ] && cat $PID_FILE | xargs pkill -P &> /dev/null; then + success + RETVAL=0 + rm -f $PID_FILE + else + failure "$PID_FILE does not exist or could not kill." + RETVAL=1 + fi + fi + RETVAL=$? + echo + return $RETVAL +} + +restart() { + stop + start +} + +# See how we were called. +case "$1" in + start|stop|restart) + $1 + ;; + status) + if [ -f $SUSE_RELEASE ]; then + echo -n "Checking for service salt-api " + checkproc $SALTAPI + rc_status -v + elif [ -f $DEBIAN_VERSION ]; then + if [ -f $LOCKFILE ]; then + RETVAL=0 + echo "salt-api is running." + else + RETVAL=1 + echo "salt-api is stopped." + fi + else + status $PROCESS + RETVAL=$? + fi + ;; + condrestart) + [ -f $LOCKFILE ] && restart || : + ;; + reload) + echo "can't reload configuration, you have to restart it" + RETVAL=$? + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart|reload}" + exit 1 + ;; +esac +exit $RETVAL diff --git a/pkg/rpm/salt-api.spec b/pkg/rpm/salt-api.spec new file mode 100644 index 00000000000..6e8b0e1e416 --- /dev/null +++ b/pkg/rpm/salt-api.spec @@ -0,0 +1,167 @@ +%if ! (0%{?rhel} >= 6 || 0%{?fedora} > 12) +%global with_python26 1 +%define pybasever 2.6 +%define __python_ver 26 +%define __python %{_bindir}/python%{?pybasever} +%endif + +%define namespace saltapi +%define eggspace salt_api + +%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} +%{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} + +Name: salt-api +Version: 0.8.3 +Release: 0%{?dist} +Summary: A web api for to access salt the parallel remote execution system + +Group: System Environment/Daemons +License: ASL 2.0 +URL: http://github.com/saltstack/salt-api +Source0: http://pypi.python.org/packages/source/s/%{name}/%{name}-%{version}.tar.gz +Source1: %{name}.service +Source2: %{name} + +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) + +BuildArch: noarch + +BuildRequires: python2-devel + +Requires: salt +Requires: python-cherrypy + + +%if ! (0%{?rhel} >= 7 || 0%{?fedora} >= 15) + +Requires(post): chkconfig +Requires(preun): chkconfig +Requires(preun): initscripts +Requires(postun): initscripts + +%else + +%if 0%{?systemd_preun:1} + +Requires(post): systemd-units +Requires(preun): systemd-units +Requires(postun): systemd-units + +%endif + +BuildRequires: systemd-units + +%endif + +%description +salt-api is a modular interface on top of Salt that can provide a variety of +entry points into a running Salt system. It can start and manage multiple +interfaces allowing a REST API to coexist with XMLRPC or even a Websocket API. + +%prep +%setup -q + +%build + + +%install +rm -rf $RPM_BUILD_ROOT +%{__python} setup.py install -O1 --root $RPM_BUILD_ROOT + +%if ! (0%{?rhel} >= 7 || 0%{?fedora} >= 15) +mkdir -p $RPM_BUILD_ROOT%{_initrddir} +install -p %{SOURCE2} $RPM_BUILD_ROOT%{_initrddir}/ +%else +mkdir -p $RPM_BUILD_ROOT%{_unitdir} +install -p -m 0644 %{SOURCE1} $RPM_BUILD_ROOT%{_unitdir}/ +%endif + +%clean +rm -rf $RPM_BUILD_ROOT + +%files +%defattr(-,root,root,-) +%doc LICENSE +%{_bindir}/%{name} +%{python_sitelib}/%{namespace}/* +%{python_sitelib}/%{eggspace}-%{version}-py?.?.egg-info +%doc %{_mandir}/man1/%{name}.1* +%doc %{_mandir}/man7/%{name}.7* + +%if ! (0%{?rhel} >= 7 || 0%{?fedora} >= 15) +%attr(0755, root, root) %{_initrddir}/%{name} +%else +%{_unitdir}/%{name}.service +%endif + +# less than RHEL 8 / Fedora 16 +# not sure if RHEL 7 will use systemd yet +%if ! (0%{?rhel} >= 7 || 0%{?fedora} >= 15) + +%preun + if [ $1 -eq 0 ] ; then + /sbin/service %{name} stop >/dev/null 2>&1 + /sbin/chkconfig --del %{name} + fi + +%post + /sbin/chkconfig --add %{name} + +%postun + if [ "$1" -ge "1" ] ; then + /sbin/service %{name} condrestart >/dev/null 2>&1 || : + fi + +%else + +%preun +%if 0%{?systemd_preun:1} + %systemd_preun %{name}.service +%else + if [ $1 -eq 0 ] ; then + # Package removal, not upgrade + /bin/systemctl --no-reload disable %{name}.service > /dev/null 2>&1 || : + /bin/systemctl stop %{name}.service > /dev/null 2>&1 || : + fi +%endif + +%post +%if 0%{?systemd_post:1} + %systemd_post %{name}.service +%else + /bin/systemctl daemon-reload &>/dev/null || : +%endif + +%postun +%if 0%{?systemd_post:1} + %systemd_postun %{name}.service +%else + /bin/systemctl daemon-reload &>/dev/null + [ $1 -gt 0 ] && /bin/systemctl try-restart %{name}.service &>/dev/null || : +%endif + +%endif + +%changelog +* Wed Jul 17 2013 Andrew Niemantsverdriet - 0.8.2-0 +- Bugfix release that fixes a compatibility issue with changes in Salt 0.15.9. +- Fixed an inconsistency with the return format for the /minions convenience URL. +- Added a dedicated URL for serving an HTML app and static media + +* Tue Apr 16 2013 Andrew Niemantsverdriet - 0.8.1-0 +- Minor bugfix version released + +* Tue Apr 16 2013 Andrew Niemantsverdriet - 0.8.0-0 +- New version released + +* Tue Feb 25 2013 Andrew Niemantsverdriet - 0.7.5-3 +- Added a more detailed decription +- Removed trailing whitespace on description. +- Added BR of python-devel + +* Tue Feb 25 2013 Andrew Niemantsverdriet - 0.7.5-2 +- Fixes as suggested by https://bugzilla.redhat.com/show_bug.cgi?id=913296# + +* Tue Feb 12 2013 Andrew Niemantsverdriet - 0.7.5-1 +- Initial package diff --git a/pkg/salt-api.service b/pkg/salt-api.service new file mode 100644 index 00000000000..7419808014c --- /dev/null +++ b/pkg/salt-api.service @@ -0,0 +1,10 @@ +[Unit] +Description=The Salt API +After=syslog.target network.target + +[Service] +Type=simple +ExecStart=/usr/bin/salt-api + +[Install] +WantedBy=multi-user.target diff --git a/pkg/salt-api.upstart b/pkg/salt-api.upstart new file mode 100644 index 00000000000..a5e87c48b93 --- /dev/null +++ b/pkg/salt-api.upstart @@ -0,0 +1,10 @@ +description "Salt API" + +start on (net-device-up + and local-filesystems + and runlevel [2345]) +stop on runlevel [!2345] + +script +exec salt-api +end script diff --git a/pkg/suse/salt-api b/pkg/suse/salt-api new file mode 120000 index 00000000000..feaa3ce9dc6 --- /dev/null +++ b/pkg/suse/salt-api @@ -0,0 +1 @@ +../rpm/salt-api \ No newline at end of file diff --git a/pkg/suse/salt-api.changes b/pkg/suse/salt-api.changes new file mode 100644 index 00000000000..50db492b803 --- /dev/null +++ b/pkg/suse/salt-api.changes @@ -0,0 +1,95 @@ +------------------------------------------------------------------- +Tue Oct 29 22:38:07 UTC 2013 - aboe76@gmail.com + +- Salt-api updated to 0.8.3 + - this will likely be the last salt-api solo release, + project is merging into main Salt project. + - fixed proper logging + - better ssl options + - improved python rest_wsgi and cherrypy support + +------------------------------------------------------------------- +Fri Oct 18 11:44:15 UTC 2013 - p.drouand@gmail.com + +- Don't support sysvinit and systemd for the same system; add conditionnal + macros to use systemd only on systems which support it and sysvinit + on other systems + +------------------------------------------------------------------- +Fri Aug 9 20:24:28 UTC 2013 - aboe76@gmail.com + +- Updated salt-api init file: + Same file as the salt-api package for Rhel/Fedora + +------------------------------------------------------------------- +Thu Jul 18 04:46:39 UTC 2013 - aboe76@gmail.com + +- Update package to 0.8.2 +- Backward incompatible needs salt 0.15.9 or greater +- Changes to rest_cherrypy: + - Fixed issue #87 which caused the Salt master's PID file to be overwritten. + - Fixed an inconsistency with the return format for the /minions convenience URL. + - Added a dedicated URL for serving an HTML app + - Added dedicated URL for serving static media + +------------------------------------------------------------------- +Sun May 12 20:18:57 UTC 2013 - aboe76@gmail.com + +- Updated package spec, for systemd unit files + according to how systemd files needs to be packaged +- fixed rpmlint about reload missing with init files + +------------------------------------------------------------------- +Tue Apr 23 19:20:42 UTC 2013 - aboe76@gmail.com + +- updated init file: + removed probe/reload/force-reload they are not supported + +------------------------------------------------------------------- +Tue Apr 23 18:10:38 UTC 2013 - aboe76@gmail.com + +- Update to salt-api 0.8.1 +- Cherrypy module fixes: +* Fixes helpful error messages when loading the module if + dependencies are missing or incorrect. +* Fixes the /login view to return a 401 instead of a 500 when + authentication fails. +* This release also includes a new plain-WSGI (no deps) REST module. This + module requires an external webserver and careful deployment -- be sure + to read the docs in full before using it. + +------------------------------------------------------------------- +Mon Apr 15 18:48:31 UTC 2013 - aboe76@gmail.com + +- Updated recommends cherrypy instead of requirement + cherrypy only needed as wsgi server if user wants + +------------------------------------------------------------------- +Sun Apr 14 14:52:34 UTC 2013 - aboe76@gmail.com + +- Updated salt-api init file + +------------------------------------------------------------------- +Tue Apr 9 18:56:15 UTC 2013 - aboe76@gmail.com + +- Updated to 0.8.0 +- New authentication login +- salt-api can now run on WSGI application +- added service file for > opensuse 12.1 +- added init file for the rest + +------------------------------------------------------------------- +Wed Jan 30 21:00:43 UTC 2013 - aboe76@gmail.com + +- Updated spec file with Suse Copyright + +------------------------------------------------------------------- +Sat Jan 26 09:19:19 UTC 2013 - aboe76@gmail.com + +- updated spec file depencies fixed include python-cherrypy for salt-ui + +------------------------------------------------------------------- +Tue Jan 22 20:28:52 UTC 2013 - aboe76@gmail.com + +- initial upload 0.7.5 + diff --git a/pkg/suse/salt-api.service b/pkg/suse/salt-api.service new file mode 120000 index 00000000000..7e505af066e --- /dev/null +++ b/pkg/suse/salt-api.service @@ -0,0 +1 @@ +../salt-api.service \ No newline at end of file diff --git a/pkg/suse/salt-api.spec b/pkg/suse/salt-api.spec new file mode 100644 index 00000000000..f4675fc2310 --- /dev/null +++ b/pkg/suse/salt-api.spec @@ -0,0 +1,118 @@ +# +# spec file for package salt-api +# +# Copyright (c) 2012 SUSE LINUX Products GmbH, Nuernberg, Germany. +# +# All modifications and additions to the file contributed by third parties +# remain the property of their copyright owners, unless otherwise agreed +# upon. The license for this file, and modifications and additions to the +# file, is the same license as for the pristine package itself (unless the +# license for the pristine package is not an Open Source License, in which +# case the license is the MIT License). An "Open Source License" is a +# license that conforms to the Open Source Definition (Version 1.9) +# published by the Open Source Initiative. + +# Please submit bugfixes or comments via http://bugs.opensuse.org/ +# + +Name: salt-api +Version: 0.8.3 +Release: 0 +License: Apache-2.0 +Summary: The api for Salt a parallel remote execution system +Url: http://saltstack.org/ +Group: System/Monitoring +Source0: http://pypi.python.org/packages/source/s/%{name}/%{name}-%{version}.tar.gz +Source1: salt-api +Source2: salt-api.service +BuildRoot: %{_tmppath}/%{name}-%{version}-build + +%if 0%{?suse_version} && 0%{?suse_version} <= 1110 +%{!?python_sitelib: %global python_sitelib %(python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} +%else +BuildArch: noarch +%endif + +BuildRequires: fdupes +BuildRequires: python-devel +BuildRequires: salt >= 0.15.9 +BuildRequires: salt-master + +Requires: salt +Requires: salt-master +Recommends: python-CherryPy +%if 0%{?suse_version} >= 1210 +BuildRequires: systemd +%{?systemd_requires} +%else +Requires(pre): %insserv_prereq +Requires(pre): %fillup_prereq +%endif + +%description +salt-api is a modular interface on top of Salt that can provide a variety of entry points into a running Salt system. + +%prep +%setup -q + +%build +python setup.py build + +%install +python setup.py install --prefix=%{_prefix} --root=%{buildroot} +%fdupes %{buildroot}%{_prefix} +# +##missing directories +%if 0%{?suse_version} < 1210 +mkdir -p %{buildroot}%{_sysconfdir}/init.d +mkdir -p %{buildroot}/%{_sbindir} +%endif +mkdir -p %{buildroot}%{_localstatedir}/log/salt +# +##init scripts +%if 0%{?suse_version} < 1210 +install -Dpm 0755 %{SOURCE1} %{buildroot}%{_sysconfdir}/init.d/salt-api +ln -sf /etc/init.d/salt-api %{buildroot}%{_sbindir}/rcsalt-api +%else +install -Dpm 644 %{SOURCE2} %{buildroot}%_unitdir/salt-api.service +%endif + +%preun +%if 0%{?_unitdir:1} +%service_del_preun salt-api.service +%else +%stop_on_removal +%endif + +%post +%if 0%{?_unitdir:1} +%service_add_post salt-api.service +%else +%fillup_and_insserv +%endif + +%postun +%if 0%{?_unitdir:1} +%service_del_postun salt-api.service +%else +%insserv_cleanup +%restart_on_update +%endif + + +%files +%defattr(-,root,root) +%doc LICENSE +%if 0%{?_unitdir:1} +%_unitdir +%else +%{_sysconfdir}/init.d/salt-api +%{_sbindir}/rcsalt-api +%endif +%{_mandir}/man1/salt-api.1.* +%{_mandir}/man7/salt-api.7.* +%{_bindir}/salt-api +%{python_sitelib}/* + + +%changelog diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index bdc3417d695..def165d322e 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -5,6 +5,7 @@ The management of salt command line utilities are stored in here # Import python libs from __future__ import print_function +import logging import os import sys @@ -27,6 +28,7 @@ from salt.exceptions import ( EauthAuthenticationError, ) +log = logging.getLogger(__name__) class SaltCMD(parsers.SaltCMDOptionParser): ''' @@ -440,3 +442,47 @@ class SaltSSH(parsers.SaltSSHOptionParser): ssh = salt.client.ssh.SSH(self.config) ssh.run() + + +class SaltAPI(parsers.OptionParser, parsers.ConfigDirMixIn, + parsers.LogLevelMixIn, parsers.PidfileMixin, parsers.DaemonMixIn, + parsers.MergeConfigMixIn): + ''' + The cli parser object used to fire up the salt api system. + ''' + __metaclass__ = parsers.OptionParserMeta + + VERSION = salt.version.__version__ + + # ConfigDirMixIn config filename attribute + _config_filename_ = 'master' + # LogLevelMixIn attributes + _default_logging_logfile_ = '/var/log/salt/api' + + def setup_config(self): + return salt.config.api_config(self.get_config_file_path()) + + def run(self): + ''' + Run the api + ''' + self.parse_args() + try: + if self.config['verify_env']: + logfile = self.config['log_file'] + if logfile is not None and not logfile.startswith('tcp://') \ + and not logfile.startswith('udp://') \ + and not logfile.startswith('file://'): + # Logfile is not using Syslog, verify + salt.utils.verify.verify_files( + [logfile], self.config['user'] + ) + except OSError as err: + log.error(err) + sys.exit(err.errno) + + self.setup_logfile_logger() + client = salt.client.netapi.NetapiClient(self.config) + self.daemonize_if_required() + self.set_pidfile() + client.run() diff --git a/salt/client/netapi.py b/salt/client/netapi.py new file mode 100644 index 00000000000..5dd05e761b4 --- /dev/null +++ b/salt/client/netapi.py @@ -0,0 +1,27 @@ +''' +The main entry point for salt-api +''' +# Import python libs +import logging +import multiprocessing + +# Import salt-api libs +import salt.loader + +logger = logging.getLogger(__name__) + +class NetapiClient(object): + ''' + ''' + def __init__(self, opts): + self.opts = opts + + def run(self): + ''' + Load and start all available api modules + ''' + netapi = salt.loader.netapi(self.opts) + for fun in netapi: + if fun.endswith('.start'): + logger.info("Starting '{0}' api module".format(fun)) + multiprocessing.Process(target=netapi[fun]).start() diff --git a/salt/config.py b/salt/config.py index 97d733888a0..53491d8efba 100644 --- a/salt/config.py +++ b/salt/config.py @@ -539,6 +539,13 @@ CLOUD_CONFIG_DEFAULTS = { 'log_granular_levels': {}, } +DEFAULT_API_OPTS = { + # ----- Salt master settings overridden by Salt-API ---------------------> + 'pidfile': '/var/run/salt-api.pid', + 'logfile': '/var/log/salt/api', + # <---- Salt master settings overridden by Salt-API ---------------------- +} + VM_CONFIG_DEFAULTS = { 'default_include': 'cloud.profiles.d/*.conf', } @@ -2122,3 +2129,16 @@ def client_config(path, env_var='SALT_CLIENT_CONFIG', defaults=None): # Return the client options _validate_opts(opts) return opts + + +def api_config(path): + ''' + Read in the salt master config file and add additional configs that + need to be stubbed out for salt-api + ''' + # Let's grab a copy of salt's master default opts + defaults = DEFAULT_MASTER_OPTS + # Let's override them with salt-api's required defaults + defaults.update(DEFAULT_API_OPTS) + + return master_config(path, defaults=defaults) diff --git a/salt/loader.py b/salt/loader.py index 63fbd1eb126..7f838b4da8e 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -451,6 +451,19 @@ def clouds(opts): return functions +def netapi(opts): + ''' + Return the network api functions + ''' + load = salt.loader._create_loader( + opts, + 'netapi', + 'netapi', + base_path=os.path.join(SALT_BASE_PATH, 'netapi'), + ) + return load.gen_functions() + + def _generate_module(name): if name in sys.modules: return diff --git a/salt/netapi/__init__.py b/salt/netapi/__init__.py new file mode 100644 index 00000000000..3bb9e408d20 --- /dev/null +++ b/salt/netapi/__init__.py @@ -0,0 +1,102 @@ +''' +Make api awesomeness +''' +# Import Python libs +import inspect + +# Import Salt libs +import salt.log # pylint: disable=W0611 +import salt.client +import salt.runner +import salt.wheel +import salt.utils +from salt.exceptions import SaltException, EauthAuthenticationError + +class NetapiClient(object): + ''' + Provide a uniform method of accessing the various client interfaces in Salt + in the form of low-data data structures. For example: + + >>> client = NetapiClient(__opts__) + >>> lowstate = {'client': 'local', 'tgt': '*', 'fun': 'test.ping', 'arg': ''} + >>> client.run(lowstate) + ''' + def __init__(self, opts): + self.opts = opts + + def run(self, low): + ''' + Execute the specified function in the specified client by passing the + lowstate + ''' + if not 'client' in low: + raise SaltException('No client specified') + + if not ('token' in low or 'eauth' in low): + raise EauthAuthenticationError( + 'No authentication credentials given') + + l_fun = getattr(self, low['client']) + f_call = salt.utils.format_call(l_fun, low) + + ret = l_fun(*f_call.get('args', ()), **f_call.get('kwargs', {})) + return ret + + def local_async(self, *args, **kwargs): + ''' + Run :ref:`execution modules ` asyncronously + + Wraps :py:meth:`salt.client.LocalClient.run_job`. + + :return: job ID + ''' + local = salt.client.get_local_client(self.opts['conf_file']) + return local.run_job(*args, **kwargs) + + def local(self, *args, **kwargs): + ''' + Run :ref:`execution modules ` syncronously + + Wraps :py:meth:`salt.client.LocalClient.cmd`. + + :return: Returns the result from the execution module + ''' + local = salt.client.get_local_client(self.opts['conf_file']) + return local.cmd(*args, **kwargs) + + def local_batch(self, *args, **kwargs): + ''' + Run :ref:`execution modules ` against batches of minions + + .. versionadded:: 0.8.4 + + Wraps :py:meth:`salt.client.LocalClient.cmd_batch` + + :return: Returns the result from the exeuction module for each batch of + returns + ''' + local = salt.client.get_local_client(self.opts['conf_file']) + return local.cmd_batch(*args, **kwargs) + + def runner(self, fun, **kwargs): + ''' + Run `runner modules ` + + Wraps :py:meth:`salt.runner.RunnerClient.low`. + + :return: Returns the result from the runner module + ''' + runner = salt.runner.RunnerClient(self.opts) + return runner.low(fun, kwargs) + + def wheel(self, fun, **kwargs): + ''' + Run :ref:`wheel modules ` + + Wraps :py:meth:`salt.wheel.WheelClient.master_call`. + + :return: Returns the result from the wheel module + ''' + kwargs['fun'] = fun + wheel = salt.wheel.Wheel(self.opts) + return wheel.master_call(**kwargs) diff --git a/salt/netapi/rest_cherrypy/__init__.py b/salt/netapi/rest_cherrypy/__init__.py new file mode 100644 index 00000000000..984ae9f05df --- /dev/null +++ b/salt/netapi/rest_cherrypy/__init__.py @@ -0,0 +1,98 @@ +''' +A script to start the CherryPy WSGI server + +This is run by ``salt-api`` and started in a multiprocess. +''' +# pylint: disable=C0103 + +# Import Python libs +import logging +import os +import signal +import sys + +# Import CherryPy without traceback so we can provide an intelligent log +# message in the __virtual__ function +try: + import cherrypy + + cpy_error = None +except ImportError as exc: + cpy_error = exc + +logger = logging.getLogger(__name__) +cpy_min = '3.2.2' + +__virtualname__ = 'rest' + +def __virtual__(): + short_name = __name__.rsplit('.')[-1] + mod_opts = __opts__.get(short_name, {}) + + if mod_opts: + # User has a rest_cherrypy section in config; assume the user wants to + # run the module and increase logging severity to be helpful + + # Everything looks good; return the module name + if not cpy_error and 'port' in mod_opts: + return True + + # CherryPy wasn't imported; explain why + if cpy_error: + from distutils.version import LooseVersion as V + + if 'cherrypy' in globals() and V(cherrypy.__version__) < V(cpy_min): + error_msg = ("Required version of CherryPy is {0} or " + "greater.".format(cpy_min)) + else: + error_msg = cpy_error + + logger.error("Not loading '%s'. Error loading CherryPy: %s", + __name__, error_msg) + + # Missing port config + if not 'port' in mod_opts: + logger.error("Not loading '%s'. 'port' not specified in config", + __name__) + + return False + +def verify_certs(*args): + ''' + Sanity checking for the specified SSL certificates + ''' + msg = ("Could not find a certificate: {0}\n" + "If you want to quickly generate a self-signed certificate, " + "use the tls.create_self_signed_cert function in Salt") + + for arg in args: + if not os.path.exists(arg): + raise Exception(msg.format(arg)) + +def start(): + ''' + Start the server loop + ''' + from . import app + root, apiopts, conf = app.get_app(__opts__) + + if not apiopts.get('disable_ssl', False): + if not 'ssl_crt' in apiopts or not 'ssl_key' in apiopts: + logger.error("Not starting '%s'. Options 'ssl_crt' and " + "'ssl_key' are required if SSL is not disabled.", + __name__) + + return None + + verify_certs(apiopts['ssl_crt'], apiopts['ssl_key']) + + cherrypy.server.ssl_module = 'builtin' + cherrypy.server.ssl_certificate = apiopts['ssl_crt'] + cherrypy.server.ssl_private_key = apiopts['ssl_key'] + + def signal_handler(*args): + cherrypy.engine.exit() + sys.exit(0) + signal.signal(signal.SIGINT, signal_handler) + + cherrypy.quickstart(root, apiopts.get('root_prefix', '/'), conf) diff --git a/salt/netapi/rest_cherrypy/app.py b/salt/netapi/rest_cherrypy/app.py new file mode 100644 index 00000000000..11a1dd50d52 --- /dev/null +++ b/salt/netapi/rest_cherrypy/app.py @@ -0,0 +1,1734 @@ +''' +A REST API for Salt +=================== + +.. versionaddedd:: Helium + +.. py:currentmodule:: salt.netapi.rest_cherrypy.app + +:depends: - CherryPy Python module +:optdepends: - ws4py Python module for websockets support. +:configuration: All authentication is done through Salt's :ref:`external auth + ` system which requires additional configuration not described + here. + + Example production-ready configuration; add to the Salt master config file: + + .. code-block:: yaml + + rest_cherrypy: + port: 8000 + ssl_crt: /etc/pki/tls/certs/localhost.crt + ssl_key: /etc/pki/tls/certs/localhost.key + + Using only a secure HTTPS connection is strongly recommended since Salt + authentication credentials will be sent over the wire. + + A self-signed certificate can be generated using the + :py:func:`~salt.modules.tls.create_self_signed_cert` function in Salt (note + the dependencies for this module). + + .. code-block:: bash + + % salt-call tls.create_self_signed_cert + + All available configuration options are detailed below. These settings + configure the CherryPy HTTP server and do not apply when using an external + server such as Apache or Nginx. + + port + **Required** + + The port for the webserver to listen on. + host : ``0.0.0.0`` + The socket interface for the HTTP server to listen on. + debug : ``False`` + Starts the web server in development mode. It will reload itself when + the underlying code is changed and will output more debugging info. + ssl_crt + The path to a SSL certificate. (See below) + ssl_key + The path to the private key for your SSL certificate. (See below) + disable_ssl + A flag to disable SSL. Warning: your Salt authentication credentials + will be sent in the clear! + webhook_disable_auth : False + The :py:class:`Webhook` URL requires authentication by default but + external services cannot always be configured to send authentication. + See the Webhook documentation for suggestions on securing this + interface. + webhook_url : /hook + Configure the URL endpoint for the :py:class:`Webhook` entry point. + thread_pool : ``100`` + The number of worker threads to start up in the pool. + socket_queue_size : ``30`` + Specify the maximum number of HTTP connections to queue. + max_request_body_size : ``1048576`` + Maximum size for the HTTP request body. + collect_stats : False + Collect and report statistics about the CherryPy server + + Reports are available via the :py:class:`Stats` URL. + static + A filesystem path to static HTML/JavaScript/CSS/image assets. + static_path : ``/static`` + The URL prefix to use when serving static assets out of the directory + specified in the ``static`` setting. + app + A filesystem path to an HTML file that will be served as a static file. + This is useful for bootstrapping a single-page JavaScript app. + app_path : ``/app`` + The URL prefix to use for serving the HTML file specified in the ``app`` + setting. This should be a simple name containing no slashes. + + Any path information after the specified path is ignored; this is + useful for apps that utilize the HTML5 history API. + root_prefix : ``/`` + A URL path to the main entry point for the application. This is useful + for serving multiple applications from the same URL. + +.. _rest_cherrypy-auth: + +Authentication +-------------- + +Authentication is performed by passing a session token with each request. +Tokens are generated via the :py:class:`Login` URL. + +The token may be sent in one of two ways: + +* Include a custom header named :mailheader:`X-Auth-Token`. +* Sent via a cookie. This option is a convenience for HTTP clients that + automatically handle cookie support (such as browsers). + +.. seealso:: You can bypass the session handling via the :py:class:`Run` URL. + +Usage +----- + +Commands are sent to a running Salt master via this module by sending HTTP +requests to the URLs detailed below. + +.. admonition:: Content negotiation + + This REST interface is flexible in what data formats it will accept as well + as what formats it will return (e.g., JSON, YAML, x-www-form-urlencoded). + + * Specify the format of data in the request body by including the + :mailheader:`Content-Type` header. + * Specify the desired data format for the response body with the + :mailheader:`Accept` header. + +Data sent in :http:method:`post` and :http:method:`put` requests must be in +the format of a list of lowstate dictionaries. This allows multiple commands to +be executed in a single HTTP request. + +.. glossary:: + + lowstate + A dictionary containing various keys that instruct Salt which command + to run, where that command lives, any parameters for that command, any + authentication credentials, what returner to use, etc. + + Salt uses the lowstate data format internally in many places to pass + command data between functions. Salt also uses lowstate for the + :ref:`LocalClient() ` Python API interface. + +The following example (in JSON format) causes Salt to execute two commands:: + + [{ + "client": "local", + "tgt": "*", + "fun": "test.fib", + "arg": ["10"] + }, + { + "client": "runner", + "fun": "jobs.lookup_jid", + "jid": "20130603122505459265" + }] + +.. admonition:: x-www-form-urlencoded + + Sending JSON or YAML in the request body is simple and most flexible, + however sending data in urlencoded format is also supported with the + caveats below. It is the default format for HTML forms, many JavaScript + libraries, and the :command:`curl` command. + + For example, the equivalent to running ``salt '*' test.ping`` is sending + ``fun=test.ping&arg&client=local&tgt=*`` in the HTTP request body. + + Caveats: + + * Only a single command may be sent per HTTP request. + * Repeating the ``arg`` parameter multiple times will cause those + parameters to be combined into a single list. + + Note, some popular frameworks and languages (notably jQuery, PHP, and + Ruby on Rails) will automatically append empty brackets onto repeated + parameters. E.g., ``arg=one``, ``arg=two`` will be sent as ``arg[]=one``, + ``arg[]=two``. This is not supported; send JSON or YAML instead. + + +.. |req_token| replace:: a session token from :py:class:`~Login`. +.. |req_accept| replace:: the desired response format. +.. |req_ct| replace:: the format of the request body. + +.. |res_ct| replace:: the format of the response body; depends on the + :mailheader:`Accept` request header. + +.. |200| replace:: success +.. |401| replace:: authentication required +.. |406| replace:: requested Content-Type not available +''' +# We need a custom pylintrc here... +# pylint: disable=W0212,E1101,C0103,R0201,W0221,W0613 + +# Import Python libs +import collections +import itertools +import functools +import logging +import json +import time +from multiprocessing import Process, Pipe + +# Import third-party libs +import cherrypy +from cherrypy.lib import cpstats +import yaml + +# Import Salt libs +import salt +import salt.auth +import salt.utils.event + +# Import salt-api libs +import salt.netapi + +logger = logging.getLogger(__name__) + +# Imports related to websocket +try: + from .tools import websockets + import event_processor + from ws4py.server.cherrypyserver import WebSocketPlugin, WebSocketTool + + HAS_WEBSOCKETS = True +except ImportError: + logger.debug('Error setting up websockets', exc_info=True) + + websockets = type('websockets', (object,), { + 'SynchronizingWebsocket': None, + }) + + HAS_WEBSOCKETS = False + + + +def salt_token_tool(): + ''' + If the custom authentication header is supplied, put it in the cookie dict + so the rest of the session-based auth works as intended + ''' + x_auth = cherrypy.request.headers.get('X-Auth-Token', None) + + # X-Auth-Token header trumps session cookie + if x_auth: + cherrypy.request.cookie['session_id'] = x_auth + +def salt_ip_verify_tool(): + ''' + If there is a list of restricted IPs, verify current + client is coming from one of those IPs. + ''' + # This is overly cumbersome and crude, + # But, it's also safe... ish... + salt_config = cherrypy.config.get('saltopts', None) + if salt_config: + cherrypy_conf = salt_config.get('rest_cherrypy', None) + if cherrypy_conf: + auth_ip_list = cherrypy_conf.get('authorized_ips', None) + if auth_ip_list: + logger.debug("Found IP list: {0}".format(auth_ip_list)) + rem_ip = cherrypy.request.headers.get('Remote-Addr', None) + logger.debug("Request from IP: {0}".format(rem_ip)) + if not rem_ip in auth_ip_list: + logger.error("Blocked IP: {0}".format(rem_ip)) + cherrypy.response.status = 403 + return { + 'status': cherrypy.response.status, + 'return': "Bad IP", + } + request = cherrypy.serving.request + cherrypy.response.headers['Access-Control-Allow-Origin'] = '*' + + +def salt_auth_tool(): + ''' + Redirect all unauthenticated requests to the login page + ''' + # Redirect to the login page if the session hasn't been authed + if not cherrypy.session.has_key('token'): + raise cherrypy.InternalRedirect('/login') + + # Session is authenticated; inform caches + cherrypy.response.headers['Cache-Control'] = 'private' + +# Be conservative in what you send +# Maps Content-Type to serialization functions; this is a tuple of tuples to +# preserve order of preference. +ct_out_map = ( + ('application/json', json.dumps), + ('application/x-yaml', functools.partial( + yaml.safe_dump, default_flow_style=False)), +) + +def hypermedia_handler(*args, **kwargs): + ''' + Determine the best output format based on the Accept header, execute the + regular handler, and transform the output to the request content type (even + if it's an error). + + :param args: Pass args through to the main handler + :param kwargs: Pass kwargs through to the main handler + ''' + # Execute the real handler. Handle or pass-through any errors we know how + # to handle (auth & HTTP errors). Reformat any errors we don't know how to + # handle as a data structure. + try: + cherrypy.response.processors = dict(ct_out_map) # handlers may modify this + ret = cherrypy.serving.request._hypermedia_inner_handler(*args, **kwargs) + except salt.exceptions.EauthAuthenticationError: + raise cherrypy.InternalRedirect('/login') + except cherrypy.CherryPyException: + raise + except Exception as exc: + import traceback + + logger.debug("Error while processing request for: %s", + cherrypy.request.path_info, + exc_info=True) + + cherrypy.response.status = 500 + + ret = { + 'status': cherrypy.response.status, + 'return': '{0}'.format(traceback.format_exc(exc)) + if cherrypy.config['debug'] + else "An unexpected error occurred"} + + # Raises 406 if requested content-type is not supported + best = cherrypy.lib.cptools.accept([i for (i, _) in ct_out_map]) + + # Transform the output from the handler into the requested output format + cherrypy.response.headers['Content-Type'] = best + out = cherrypy.response.processors[best] + return out(ret) + + +def hypermedia_out(): + ''' + Determine the best handler for the requested content type + + Wrap the normal handler and transform the output from that handler into the + requested content type + ''' + request = cherrypy.serving.request + request._hypermedia_inner_handler = request.handler + request.handler = hypermedia_handler + + cherrypy.response.headers['Access-Control-Allow-Origin'] = '*' + + +@functools.wraps +def process_request_body(fn): + ''' + A decorator to skip a processor function if process_request_body is False + ''' + def wrapped(*args, **kwargs): + if cherrypy.request.process_request_body != False: + fn(*args, **kwargs) + return wrapped + + +def urlencoded_processor(entity): + ''' + Accept x-www-form-urlencoded data (run through CherryPy's formatter) + and reformat it into a Low State data structure. + + Since we can't easily represent complicated data structures with + key-value pairs, any more complicated requirements (e.g. compound + commands) must instead be delivered via JSON or YAML. + + For example:: + + curl -si localhost:8000 -d client=local -d tgt='*' \\ + -d fun='test.kwarg' -d arg='one=1' -d arg='two=2' + + :param entity: raw POST data + ''' + # First call out to CherryPy's default processor + cherrypy._cpreqbody.process_urlencoded(entity) + cherrypy.serving.request.unserialized_data = entity.params + + +@process_request_body +def json_processor(entity): + ''' + Unserialize raw POST data in JSON format to a Python data structure. + + :param entity: raw POST data + ''' + body = entity.fp.read() + try: + cherrypy.serving.request.unserialized_data = json.loads(body) + except ValueError: + raise cherrypy.HTTPError(400, 'Invalid JSON document') + + +@process_request_body +def yaml_processor(entity): + ''' + Unserialize raw POST data in YAML format to a Python data structure. + + :param entity: raw POST data + ''' + body = entity.fp.read() + try: + cherrypy.serving.request.unserialized_data = yaml.safe_load(body) + except ValueError: + raise cherrypy.HTTPError(400, 'Invalid YAML document') + + +@process_request_body +def text_processor(entity): + ''' + Attempt to unserialize plain text as JSON + + Some large services still send JSON with a text/plain Content-Type. Those + services are bad and should feel bad. + + :param entity: raw POST data + ''' + body = entity.fp.read() + try: + cherrypy.serving.request.unserialized_data = json.loads(body) + except ValueError: + cherrypy.serving.request.unserialized_data = body + + +def hypermedia_in(): + ''' + Unserialize POST/PUT data of a specified Content-Type. + + The following custom processors all are intended to format Low State data + and will place that data structure into the request object. + + :raises HTTPError: if the request contains a Content-Type that we do not + have a processor for + ''' + # Be liberal in what you accept + ct_in_map = { + 'application/x-www-form-urlencoded': urlencoded_processor, + 'application/json': json_processor, + 'application/x-yaml': yaml_processor, + 'text/yaml': yaml_processor, + 'text/plain': text_processor, + } + + # Do not process the body for POST requests that have specified no content + # or have not specified Content-Length + if (cherrypy.request.method.upper() == 'POST' + and cherrypy.request.headers.get('Content-Length', '0') == '0'): + cherrypy.request.process_request_body = False + + cherrypy.request.body.processors.clear() + cherrypy.request.body.default_proc = cherrypy.HTTPError( + 406, 'Content type not supported') + cherrypy.request.body.processors = ct_in_map + + +def lowdata_fmt(): + ''' + Validate and format lowdata from incoming unserialized request data + + This tool requires that the hypermedia_in tool has already been run. + ''' + if cherrypy.request.method.upper() != 'POST': + return + + # TODO: call lowdata validation routines from here + + data = cherrypy.request.unserialized_data + + if cherrypy.request.headers['Content-Type'] == 'application/x-www-form-urlencoded': + # Make the 'arg' param a list if not already + if 'arg' in data and not isinstance(data['arg'], list): + data['arg'] = [data['arg']] + + # Finally, make a Low State and put it in request + cherrypy.request.lowstate = [data] + else: + cherrypy.serving.request.lowstate = data + + +cherrypy.tools.salt_token = cherrypy.Tool('on_start_resource', + salt_token_tool, priority=55) +cherrypy.tools.salt_auth = cherrypy.Tool('before_request_body', + salt_auth_tool, priority=60) +cherrypy.tools.hypermedia_in = cherrypy.Tool('before_request_body', + hypermedia_in) +cherrypy.tools.lowdata_fmt = cherrypy.Tool('before_handler', + lowdata_fmt, priority=40) +cherrypy.tools.hypermedia_out = cherrypy.Tool('before_handler', + hypermedia_out) +cherrypy.tools.salt_ip_verify = cherrypy.Tool('before_handler', + salt_ip_verify_tool) + + +############################################################################### + + +class LowDataAdapter(object): + ''' + The primary entry point to Salt's REST API + + ''' + exposed = True + + _cp_config = { + 'tools.sessions.on': True, + 'tools.sessions.timeout': 60 * 10, # 10 hours + + # 'tools.autovary.on': True, + + 'tools.hypermedia_out.on': True, + 'tools.hypermedia_in.on': True, + 'tools.lowdata_fmt.on': True, + 'tools.salt_ip_verify.on': True, + } + + def __init__(self): + self.opts = cherrypy.config['saltopts'] + self.api = salt.netapi.NetapiClient(self.opts) + + def exec_lowstate(self, client=None, token=None): + ''' + Pull a Low State data structure from request and execute the low-data + chunks through Salt. The low-data chunks will be updated to include the + authorization token for the current session. + ''' + lowstate = cherrypy.request.lowstate + + # Release the session lock before executing any potentially + # long-running Salt commands. This allows different threads to execute + # Salt commands concurrently without blocking. + cherrypy.session.release_lock() + + # if the lowstate loaded isn't a list, lets notify the client + if type(lowstate) != list: + raise cherrypy.HTTPError(400, 'Lowstates must be a list') + + # Make any requested additions or modifications to each lowstate, then + # execute each one and yield the result. + for chunk in lowstate: + if token: + chunk['token'] = token + + if client: + chunk['client'] = client + + # Make any 'arg' params a list if not already. + # This is largely to fix a deficiency in the urlencoded format. + if 'arg' in chunk and not isinstance(chunk['arg'], list): + chunk['arg'] = [chunk['arg']] + + ret = self.api.run(chunk) + + # Sometimes Salt gives us a return and sometimes an iterator + if isinstance(ret, collections.Iterator): + for i in ret: + yield i + else: + yield ret + + def GET(self): + ''' + An explanation of the API with links of where to go next + + .. http:get:: / + + :reqheader Accept: |req_accept| + + :status 200: |200| + :status 401: |401| + :status 406: |406| + + **Example request**:: + + % curl -i localhost:8000 + + .. code-block:: http + + GET / HTTP/1.1 + Host: localhost:8000 + Accept: application/json + + **Example response**: + + .. code-block:: http + + HTTP/1.1 200 OK + Content-Type: application/json + ''' + import inspect + + # Grab all available client interfaces + clients = [name for name, _ in inspect.getmembers(salt.netapi.NetapiClient, + predicate=inspect.ismethod) if not name.startswith('__')] + clients.remove('run') # run method calls client interfaces + + return { + 'return': "Welcome", + 'clients': clients, + } + + @cherrypy.tools.salt_token() + @cherrypy.tools.salt_auth() + def POST(self, **kwargs): + ''' + Send one or more Salt commands in the request body + + .. http:post:: / + + :reqheader X-Auth-Token: |req_token| + :reqheader Accept: |req_accept| + :reqheader Content-Type: |req_ct| + + :resheader Content-Type: |res_ct| + + :status 200: |200| + :status 401: |401| + :status 406: |406| + + :term:`lowstate` data describing Salt commands must be sent in the + request body. + + **Example request**:: + + % curl -si https://localhost:8000 \\ + -H "Accept: application/x-yaml" \\ + -H "X-Auth-Token: d40d1e1e" \\ + -d client=local \\ + -d tgt='*' \\ + -d fun='test.ping' \\ + -d arg + + .. code-block:: http + + POST / HTTP/1.1 + Host: localhost:8000 + Accept: application/x-yaml + X-Auth-Token: d40d1e1e + Content-Length: 36 + Content-Type: application/x-www-form-urlencoded + + fun=test.ping&arg&client=local&tgt=* + + **Example response**: + + .. code-block:: http + + HTTP/1.1 200 OK + Content-Length: 200 + Allow: GET, HEAD, POST + Content-Type: application/x-yaml + + return: + - ms-0: true + ms-1: true + ms-2: true + ms-3: true + ms-4: true + ''' + return { + 'return': list(self.exec_lowstate( + token=cherrypy.session.get('token'))) + } + + +class Minions(LowDataAdapter): + _cp_config = dict(LowDataAdapter._cp_config, **{ + 'tools.salt_token.on': True, + 'tools.salt_auth.on': True, + }) + + def GET(self, mid=None): + ''' + A convenience URL for getting lists of minions or getting minion + details + + .. http:get:: /minions/(mid) + + :reqheader X-Auth-Token: |req_token| + :reqheader Accept: |req_accept| + + :status 200: |200| + :status 401: |401| + :status 406: |406| + + **Example request**:: + + % curl -i localhost:8000/minions/ms-3 + + .. code-block:: http + + GET /minions/ms-3 HTTP/1.1 + Host: localhost:8000 + Accept: application/x-yaml + + **Example response**: + + .. code-block:: http + + HTTP/1.1 200 OK + Content-Length: 129005 + Content-Type: application/x-yaml + + return: + - ms-3: + grains.items: + ... + ''' + cherrypy.request.lowstate = [{ + 'client': 'local', 'tgt': mid or '*', 'fun': 'grains.items', + }] + return { + 'return': list(self.exec_lowstate( + token=cherrypy.session.get('token'))), + } + + def POST(self, **kwargs): + ''' + Start an execution command and immediately return the job id + + .. http:post:: /minions + + :reqheader X-Auth-Token: |req_token| + :reqheader Accept: |req_accept| + :reqheader Content-Type: |req_ct| + + :resheader Content-Type: |res_ct| + + :status 200: |200| + :status 401: |401| + :status 406: |406| + + :term:`lowstate` data describing Salt commands must be sent in the + request body. The ``client`` option will be set to + :py:meth:`~salt.client.LocalClient.local_async`. + + **Example request**:: + + % curl -sSi localhost:8000/minions \\ + -H "Accept: application/x-yaml" \\ + -d tgt='*' \\ + -d fun='status.diskusage' + + .. code-block:: http + + POST /minions HTTP/1.1 + Host: localhost:8000 + Accept: application/x-yaml + Content-Length: 26 + Content-Type: application/x-www-form-urlencoded + + tgt=*&fun=status.diskusage + + **Example response**: + + .. code-block:: http + + HTTP/1.1 202 Accepted + Content-Length: 86 + Content-Type: application/x-yaml + + return: + - jid: '20130603122505459265' + minions: [ms-4, ms-3, ms-2, ms-1, ms-0] + _links: + jobs: + - href: /jobs/20130603122505459265 + ''' + job_data = list(self.exec_lowstate(client='local_async', + token=cherrypy.session.get('token'))) + + cherrypy.response.status = 202 + return { + 'return': job_data, + '_links': { + 'jobs': [{'href': '/jobs/{0}'.format(i['jid'])} + for i in job_data if i], + }, + } + + +class Jobs(LowDataAdapter): + _cp_config = dict(LowDataAdapter._cp_config, **{ + 'tools.salt_token.on': True, + 'tools.salt_auth.on': True, + }) + + def GET(self, jid=None): + ''' + A convenience URL for getting lists of previously run jobs or getting + the return from a single job + + .. http:get:: /jobs/(jid) + + List jobs or show a single job from the job cache. + + :status 200: |200| + :status 401: |401| + :status 406: |406| + + **Example request**:: + + % curl -i localhost:8000/jobs + + .. code-block:: http + + GET /jobs HTTP/1.1 + Host: localhost:8000 + Accept: application/x-yaml + + **Example response**: + + .. code-block:: http + + HTTP/1.1 200 OK + Content-Length: 165 + Content-Type: application/x-yaml + + return: + - '20121130104633606931': + Arguments: + - '3' + Function: test.fib + Start Time: 2012, Nov 30 10:46:33.606931 + Target: jerry + Target-type: glob + + **Example request**:: + + % curl -i localhost:8000/jobs/20121130104633606931 + + .. code-block:: http + + GET /jobs/20121130104633606931 HTTP/1.1 + Host: localhost:8000 + Accept: application/x-yaml + + **Example response**: + + .. code-block:: http + + HTTP/1.1 200 OK + Content-Length: 73 + Content-Type: application/x-yaml + + info: + - Arguments: + - '3' + Function: test.fib + Minions: + - jerry + Start Time: 2012, Nov 30 10:46:33.606931 + Target: '*' + Target-type: glob + User: saltdev + jid: '20121130104633606931' + return: + - jerry: + - - 0 + - 1 + - 1 + - 2 + - 6.9141387939453125e-06 + ''' + lowstate = [{ + 'client': 'runner', + 'fun': 'jobs.lookup_jid' if jid else 'jobs.list_jobs', + 'jid': jid, + }] + + if jid: + lowstate.append({ + 'client': 'runner', + 'fun': 'jobs.list_job', + 'jid': jid, + }) + + cherrypy.request.lowstate = lowstate + job_ret_info = list(self.exec_lowstate( + token=cherrypy.session.get('token'))) + + ret = {} + if jid: + job_ret, job_info = job_ret_info + ret['info'] = [job_info] + else: + job_ret = job_ret_info[0] + + ret['return'] = [job_ret] + return ret + + +class Login(LowDataAdapter): + ''' + Log in to recieve a session token + + :ref:`Authentication information `. + ''' + + def __init__(self, *args, **kwargs): + super(Login, self).__init__(*args, **kwargs) + + self.auth = salt.auth.Resolver(self.opts) + + def GET(self): + ''' + Present the login interface + + .. http:get:: /login + + An explanation of how to log in. + + :status 200: |200| + :status 401: |401| + :status 406: |406| + + **Example request**:: + + % curl -i localhost:8000/login + + .. code-block:: http + + GET /login HTTP/1.1 + Host: localhost:8000 + Accept: text/html + + **Example response**: + + .. code-block:: http + + HTTP/1.1 200 OK + Content-Type: text/html + ''' + cherrypy.response.headers['WWW-Authenticate'] = 'Session' + + return { + 'status': cherrypy.response.status, + 'return': "Please log in", + } + + def POST(self, **kwargs): + ''' + :ref:`Authenticate ` against Salt's eauth system + + .. http:post:: /login + + :reqheader X-Auth-Token: |req_token| + :reqheader Accept: |req_accept| + :reqheader Content-Type: |req_ct| + + :form eauth: the eauth backend configured for the user + :form username: username + :form password: password + + :status 200: |200| + :status 401: |401| + :status 406: |406| + + **Example request**:: + + % curl -si localhost:8000/login \\ + -H "Accept: application/json" \\ + -d username='saltuser' \\ + -d password='saltpass' \\ + -d eauth='pam' + + .. code-block:: http + + POST / HTTP/1.1 + Host: localhost:8000 + Content-Length: 42 + Content-Type: application/x-www-form-urlencoded + Accept: application/json + + username=saltuser&password=saltpass&eauth=pam + + **Example response**: + + .. code-block:: http + + HTTP/1.1 200 OK + Content-Type: application/json + Content-Length: 206 + X-Auth-Token: 6d1b722e + Set-Cookie: session_id=6d1b722e; expires=Sat, 17 Nov 2012 03:23:52 GMT; Path=/ + + {"return": { + "token": "6d1b722e", + "start": 1363805943.776223, + "expire": 1363849143.776224, + "user": "saltuser", + "eauth": "pam", + "perms": [ + "grains.*", + "status.*", + "sys.*", + "test.*" + ] + }} + ''' + # the urlencoded_processor will wrap this in a list + if isinstance(cherrypy.serving.request.lowstate, list): + creds = cherrypy.serving.request.lowstate[0] + else: + creds = cherrypy.serving.request.lowstate + + token = self.auth.mk_token(creds) + if not 'token' in token: + raise cherrypy.HTTPError(401, + 'Could not authenticate using provided credentials') + + cherrypy.response.headers['X-Auth-Token'] = cherrypy.session.id + cherrypy.session['token'] = token['token'] + cherrypy.session['timeout'] = (token['expire'] - token['start']) / 60 + + # Grab eauth config for the current backend for the current user + try: + perms = self.opts['external_auth'][token['eauth']][token['name']] + except (AttributeError, IndexError): + logger.debug("Configuration for external_auth malformed for "\ + "eauth '{0}', and user '{1}'." + .format(token.get('eauth'), token.get('name')), exc_info=True) + raise cherrypy.HTTPError(500, + 'Configuration for external_auth could not be read.') + + return {'return': [{ + 'token': cherrypy.session.id, + 'expire': token['expire'], + 'start': token['start'], + 'user': token['name'], + 'eauth': token['eauth'], + 'perms': perms, + }]} + + +class Logout(LowDataAdapter): + _cp_config = dict(LowDataAdapter._cp_config, **{ + 'tools.salt_token.on': True, + 'tools.salt_auth.on': True, + }) + + def POST(self): + ''' + Destroy the currently active session and expire the session cookie + ''' + cherrypy.lib.sessions.expire() # set client-side to expire + cherrypy.session.regenerate() # replace server-side with new + + return {'return': "Your token has been cleared"} + + +class Run(LowDataAdapter): + _cp_config = dict(LowDataAdapter._cp_config, **{ + 'tools.sessions.on': False, + }) + + def POST(self, **kwargs): + ''' + Run commands bypassing the :ref:`normal session handling + ` + + .. http:post:: /run + + This entry point is primarily for "one-off" commands. Each request + must pass full Salt authentication credentials. Otherwise this URL + is identical to the :py:meth:`root URL (/) `. + + :term:`lowstate` data describing Salt commands must be sent in the + request body. + + :status 200: |200| + :status 401: |401| + :status 406: |406| + + **Example request**:: + + % curl -sS localhost:8000/run \\ + -H 'Accept: application/x-yaml' \\ + -d client='local' \\ + -d tgt='*' \\ + -d fun='test.ping' \\ + -d username='saltdev' \\ + -d password='saltdev' \\ + -d eauth='pam' + + .. code-block:: http + + POST /run HTTP/1.1 + Host: localhost:8000 + Accept: application/x-yaml + Content-Length: 75 + Content-Type: application/x-www-form-urlencoded + + client=local&tgt=*&fun=test.ping&username=saltdev&password=saltdev&eauth=pam + + **Example response**: + + .. code-block:: http + + HTTP/1.1 200 OK + Content-Length: 73 + Content-Type: application/x-yaml + + return: + - ms-0: true + ms-1: true + ms-2: true + ms-3: true + ms-4: true + ''' + return { + 'return': list(self.exec_lowstate()), + } + + +class Events(object): + ''' + Expose the Salt event bus + + The event bus on the Salt master exposes a large variety of things, notably + when executions are started on the master and also when minions ultimately + return their results. This URL provides a real-time window into a running + Salt infrastructure. + + .. seealso:: :ref:`events` + ''' + exposed = True + + _cp_config = dict(LowDataAdapter._cp_config, **{ + 'response.stream': True, + 'tools.encode.encoding': 'utf-8', + + # Auth handled manually below + 'tools.salt_token.on': True, + 'tools.salt_auth.on': False, + + 'tools.hypermedia_in.on': False, + 'tools.hypermedia_out.on': False, + }) + + def __init__(self): + self.opts = cherrypy.config['saltopts'] + self.auth = salt.auth.LoadAuth(self.opts) + + def GET(self, token=None): + ''' + An HTTP stream of the Salt master event bus + + This stream is formatted per the Server Sent Events (SSE) spec. Each + event is formatted as JSON. + + Browser clients currently lack Cross-origin resource sharing (CORS) + support for the ``EventSource()`` API. Cross-domain requests from a + browser may instead pass the :mailheader:`X-Auth-Token` value as an URL + parameter:: + + % curl -NsS localhost:8000/events/6d1b722e + + .. http:get:: /events + + :status 200: |200| + :status 401: |401| + :status 406: |406| + + **Example request**:: + + % curl -NsS localhost:8000/events + + .. code-block:: http + + GET /events HTTP/1.1 + Host: localhost:8000 + + **Example response**: + + .. code-block:: http + + HTTP/1.1 200 OK + Connection: keep-alive + Cache-Control: no-cache + Content-Type: text/event-stream;charset=utf-8 + + retry: 400 + data: {'tag': '', 'data': {'minions': ['ms-4', 'ms-3', 'ms-2', 'ms-1', 'ms-0']}} + + data: {'tag': '20130802115730568475', 'data': {'jid': '20130802115730568475', 'return': True, 'retcode': 0, 'success': True, 'cmd': '_return', 'fun': 'test.ping', 'id': 'ms-1'}} + + The event stream can be easily consumed via JavaScript: + + .. code-block:: javascript + + # Note, you must be authenticated! + var source = new EventSource('/events'); + source.onopen = function() { console.debug('opening') }; + source.onerror = function(e) { console.debug('error!', e) }; + source.onmessage = function(e) { console.debug(e.data) }; + + It is also possible to consume the stream via the shell. + + Records are separated by blank lines; the ``data:`` and ``tag:`` + prefixes will need to be removed manually before attempting to + unserialize the JSON. + + curl's ``-N`` flag turns off input buffering which is required to + process the stream incrementally. + + Here is a basic example of printing each event as it comes in: + + .. code-block:: bash + + % curl -NsS localhost:8000/events |\\ + while IFS= read -r line ; do + echo $line + done + + Here is an example of using awk to filter events based on tag: + + .. code-block:: bash + + % curl -NsS localhost:8000/events |\\ + awk ' + BEGIN { RS=""; FS="\\n" } + $1 ~ /^tag: salt\/job\/[0-9]+\/new$/ { print $0 } + ' + tag: salt/job/20140112010149808995/new + data: {"tag": "salt/job/20140112010149808995/new", "data": {"tgt_type": "glob", "jid": "20140112010149808995", "tgt": "jerry", "_stamp": "2014-01-12_01:01:49.809617", "user": "shouse", "arg": [], "fun": "test.ping", "minions": ["jerry"]}} + tag: 20140112010149808995 + data: {"tag": "20140112010149808995", "data": {"fun_args": [], "jid": "20140112010149808995", "return": true, "retcode": 0, "success": true, "cmd": "_return", "_stamp": "2014-01-12_01:01:49.819316", "fun": "test.ping", "id": "jerry"}} + ''' + # Pulling the session token from an URL param is a workaround for + # browsers not supporting CORS in the EventSource API. + if token: + orig_sesion, _ = cherrypy.session.cache.get(token, ({}, None)) + salt_token = orig_sesion.get('token') + else: + salt_token = cherrypy.session.get('token') + + # Manually verify the token + if not salt_token or not self.auth.get_tok(salt_token): + raise cherrypy.InternalRedirect('/login') + + # Release the session lock before starting the long-running response + cherrypy.session.release_lock() + + cherrypy.response.headers['Content-Type'] = 'text/event-stream' + cherrypy.response.headers['Cache-Control'] = 'no-cache' + cherrypy.response.headers['Connection'] = 'keep-alive' + + def listen(): + event = salt.utils.event.get_event('master', opts=self.opts) + stream = event.iter_events(full=True) + + yield u'retry: {0}\n'.format(400) + + while True: + data = stream.next() + yield u'tag: {0}\n'.format(data.get('tag', '')) + yield u'data: {0}\n\n'.format(json.dumps(data)) + + return listen() + + +class WebsocketEndpoint(object): + ''' + Open a WebSocket connection to Salt's event bus + + The event bus on the Salt master exposes a large variety of things, notably + when executions are started on the master and also when minions ultimately + return their results. This URL provides a real-time window into a running + Salt infrastructure. Uses websocket as the transport mechanism. + + .. seealso:: :ref:`events` + ''' + exposed = True + + _cp_config = dict(LowDataAdapter._cp_config, **{ + 'response.stream': True, + 'tools.encode.encoding': 'utf-8', + + # Auth handled manually below + 'tools.salt_token.on': True, + 'tools.salt_auth.on': False, + + 'tools.hypermedia_in.on': False, + 'tools.hypermedia_out.on': False, + 'tools.websocket.on': True, + 'tools.websocket.handler_cls': websockets.SynchronizingWebsocket, + }) + + def __init__(self): + self.opts = cherrypy.config['saltopts'] + self.auth = salt.auth.LoadAuth(self.opts) + + def GET(self, token=None, **kwargs): + ''' + Return a websocket connection of Salt's event stream + + .. http:get:: /ws/(token) + + :query format_events: The event stream will undergo server-side + formatting if the ``format_events`` URL parameter is included + in the request. This can be useful to avoid formatting on the + client-side:: + + curl -NsS <...snip...> localhost:8000/ws?format_events + + :reqheader X-Auth-Token: an authentication token from + :py:class:`~Login`. + + :status 101: switching to the websockets protocol + :status 401: |401| + :status 406: |406| + + **Example request**:: + + curl -NsS \\ + -H 'X-Auth-Token: ffedf49d' \\ + -H 'Host: localhost:8000' \\ + -H 'Connection: Upgrade' \\ + -H 'Upgrade: websocket' \\ + -H 'Origin: http://localhost:8000' \\ + -H 'Sec-WebSocket-Version: 13' \\ + -H 'Sec-WebSocket-Key: '"$(echo -n $RANDOM | base64)" \\ + localhost:8000/ws + + .. code-block:: http + + GET /ws HTTP/1.1 + Connection: Upgrade + Upgrade: websocket + Host: localhost:8000 + Origin: http://localhost:8000 + Sec-WebSocket-Version: 13 + Sec-WebSocket-Key: s65VsgHigh7v/Jcf4nXHnA== + X-Auth-Token: ffedf49d + + **Example response**: + + .. code-block:: http + + HTTP/1.1 101 Switching Protocols + Upgrade: websocket + Connection: Upgrade + Sec-WebSocket-Accept: mWZjBV9FCglzn1rIKJAxrTFlnJE= + Sec-WebSocket-Version: 13 + + An authentication token **may optionally** be passed as part of the URL + for browsers that cannot be configured to send the authentication + header or cookie:: + + curl -NsS <...snip...> localhost:8000/ws/ffedf49d + + The event stream can be easily consumed via JavaScript: + + .. code-block:: javascript + + // Note, you must be authenticated! + var source = new Websocket('ws://localhost:8000/ws/d0ce6c1a'); + source.onerror = function(e) { console.debug('error!', e); }; + source.onmessage = function(e) { console.debug(e.data); }; + + source.send('websocket client ready') + + source.close(); + + Or via Python, using the Python module + `websocket-client `_ for example. + + .. code-block:: python + + # Note, you must be authenticated! + + from websocket import create_connection + + ws = create_connection('ws://localhost:8000/ws/d0ce6c1a') + ws.send('websocket client ready') + + # Look at https://pypi.python.org/pypi/websocket-client/ for more examples. + while listening_to_events: + print ws.recv() + + ws.close() + + Above examples show how to establish a websocket connection to Salt and + activating real time updates from Salt's event stream by signaling + ``websocket client ready``. + ''' + # Pulling the session token from an URL param is a workaround for + # browsers not supporting CORS in the EventSource API. + if token: + orig_sesion, _ = cherrypy.session.cache.get(token, ({}, None)) + salt_token = orig_sesion.get('token') + else: + salt_token = cherrypy.session.get('token') + + # Manually verify the token + if not salt_token or not self.auth.get_tok(salt_token): + raise cherrypy.HTTPError(401) + + # Release the session lock before starting the long-running response + cherrypy.session.release_lock() + + ''' + A handler is the server side end of the websocket connection. + Each request spawns a new instance of this handler + ''' + handler = cherrypy.request.ws_handler + + def event_stream(handler, pipe): + pipe.recv() # blocks until send is called on the parent end of this pipe. + + event = salt.utils.event.get_event('master', opts=self.opts) + stream = event.iter_events(full=True) + SaltInfo = event_processor.SaltInfo(handler) + while True: + data = stream.next() + if data: + try: #work around try to decode catch unicode errors + if 'format_events' in kwargs: + SaltInfo.process(data, salt_token, self.opts) + else: + handler.send('data: {0}\n\n'.format(json.dumps(data)), False) + except UnicodeDecodeError as ex: + logger.error("Error: Salt event has non UTF-8 data:\n{0}".format(data)) + time.sleep(0.1) + + parent_pipe, child_pipe = Pipe() + handler.pipe = parent_pipe + handler.opts = self.opts + # Process to handle async push to a client. + # Each GET request causes a process to be kicked off. + proc = Process(target=event_stream, args=(handler,child_pipe)) + proc.start() + + +class Webhook(object): + ''' + A generic web hook entry point that fires an event on Salt's event bus + + External services can POST data to this URL to trigger an event in Salt. + For example, Amazon SNS, Jenkins-CI or Travis-CI, or GitHub web hooks. + + .. note:: Be mindful of security + + Salt's Reactor can run any code. A Reactor SLS that responds to a hook + event is responsible for validating that the event came from a trusted + source and contains valid data. + + **This is a generic interface and securing it is up to you!** + + This URL requires authentication however not all external services can + be configured to authenticate. For this reason authentication can be + selectively disabled for this URL. Follow best practices -- always use + SSL, pass a secret key, configure the firewall to only allow traffic + from a known source, etc. + + The event data is taken from the request body. The + :mailheader:`Content-Type` header is respected for the payload. + + The event tag is prefixed with ``salt/netapi/hook`` and the URL path is + appended to the end. For example, a ``POST`` request sent to + ``/hook/mycompany/myapp/mydata`` will produce a Salt event with the tag + ``salt/netapi/hook/mycompany/myapp/mydata``. + + The following is an example ``.travis.yml`` file to send notifications to + Salt of successful test runs: + + .. code-block:: yaml + + language: python + script: python -m unittest tests + after_success: + - 'curl -sS http://saltapi-url.example.com:8000/hook/travis/build/success -d branch="${TRAVIS_BRANCH}" -d commit="${TRAVIS_COMMIT}"' + + .. seealso:: :ref:`events`, :ref:`reactor` + ''' + exposed = True + tag_base = ['salt', 'netapi', 'hook'] + + _cp_config = dict(LowDataAdapter._cp_config, **{ + # Don't do any lowdata processing on the POST data + 'tools.lowdata_fmt.on': True, + + # Auth can be overridden in __init__(). + 'tools.salt_token.on': True, + 'tools.salt_auth.on': True, + }) + + def __init__(self): + self.opts = cherrypy.config['saltopts'] + self.event = salt.utils.event.get_event('master', opts=self.opts) + + if cherrypy.config['apiopts'].get('webhook_disable_auth'): + self._cp_config['tools.salt_token.on'] = False + self._cp_config['tools.salt_auth.on'] = False + + def POST(self, *args, **kwargs): + ''' + Fire an event in Salt with a custom event tag and data + + .. http:post:: /hook + + :status 200: |200| + :status 401: |401| + :status 406: |406| + :status 413: request body is too large + + **Example request**:: + + % curl -sS localhost:8000/hook -d foo='Foo!' -d bar='Bar!' + + .. code-block:: http + + POST /hook HTTP/1.1 + Host: localhost:8000 + Content-Length: 16 + Content-Type: application/x-www-form-urlencoded + + foo=Foo&bar=Bar! + + **Example response**: + + .. code-block:: http + + HTTP/1.1 200 OK + Content-Length: 14 + Content-Type: application/json + + {"success": true} + + As a practical example, an internal continuous-integration build + server could send an HTTP POST request to the URL + ``http://localhost:8000/hook/mycompany/build/success`` which contains + the result of a build and the SHA of the version that was built as + JSON. That would then produce the following event in Salt that could be + used to kick off a deployment via Salt's Reactor:: + + Event fired at Fri Feb 14 17:40:11 2014 + ************************* + Tag: salt/netapi/hook/mycompany/build/success + Data: + {'_stamp': '2014-02-14_17:40:11.440996', + 'headers': { + 'X-My-Secret-Key': 'F0fAgoQjIT@W', + 'Content-Length': '37', + 'Content-Type': 'application/json', + 'Host': 'localhost:8000', + 'Remote-Addr': '127.0.0.1'}, + 'post': {'revision': 'aa22a3c4b2e7', 'result': True}} + + Salt's Reactor could listen for the event: + + .. code-block:: yaml + + reactor: + - 'salt/netapi/hook/mycompany/build/*': + - /srv/reactor/react_ci_builds.sls + + And finally deploy the new build: + + .. code-block:: yaml + + {% set secret_key = data.get('headers', {}).get('X-My-Secret-Key') %} + {% set build = data.get('post', {}) %} + + {% if secret_key == 'F0fAgoQjIT@W' and build.result == True %} + deploy_my_app: + cmd.state.sls: + - tgt: 'application*' + - arg: + - myapp.deploy + - kwarg: + pillar: + revision: {{ revision }} + {% endif %} + ''' + tag = '/'.join(itertools.chain(self.tag_base, args)) + data = cherrypy.serving.request.unserialized_data + headers = dict(cherrypy.request.headers) + + ret = self.event.fire_event({ + 'post': data, + 'headers': headers, + }, tag) + return {'success': ret} + + +class Stats(object): + ''' + Expose statistics on the running CherryPy server + ''' + exposed = True + + _cp_config = dict(LowDataAdapter._cp_config, **{ + 'tools.salt_token.on': True, + 'tools.salt_auth.on': True, + }) + + def GET(self): + ''' + Return a dump of statistics collected from the CherryPy server + + .. http:get:: /stats + + :reqheader X-Auth-Token: |req_token| + :reqheader Accept: |req_accept| + + :resheader Content-Type: |res_ct| + + :status 200: |200| + :status 401: |401| + :status 406: |406| + ''' + if hasattr(logging, 'statistics'): + return cpstats.extrapolate_statistics(logging.statistics) + + return {} + + +class App(object): + exposed = True + def GET(self, *args): + ''' + Serve a single static file ignoring the remaining path + + This is useful in combination with a browser-based app using the HTML5 + history API. + + .. http::get:: /app + + :reqheader X-Auth-Token: |req_token| + + :status 200: |200| + :status 401: |401| + ''' + apiopts = cherrypy.config['apiopts'] + return cherrypy.lib.static.serve_file(apiopts['app']) + + +class API(object): + ''' + Collect configuration and URL map for building the CherryPy app + ''' + url_map = { + 'index': LowDataAdapter, + 'login': Login, + 'logout': Logout, + 'minions': Minions, + 'run': Run, + 'jobs': Jobs, + 'events': Events, + 'stats': Stats, + } + + def _setattr_url_map(self): + for url, cls in self.url_map.items(): + setattr(self, url, cls()) + + def _update_url_map(self): + ''' + Assemble any dynamic or configurable URLs + ''' + if HAS_WEBSOCKETS: + self.url_map.update({ + 'ws': WebsocketEndpoint, + }) + + # Allow the Webhook URL to be overridden from the conf. + self.url_map.update({ + self.apiopts.get('webhook_url', 'hook').lstrip('/'): Webhook, + }) + + # Enable the single-page JS app URL. + if 'app' in self.apiopts: + self.url_map.update({ + self.apiopts.get('app_path', 'app').lstrip('/'): App, + }) + + def __init__(self): + self.opts = cherrypy.config['saltopts'] + self.apiopts = cherrypy.config['apiopts'] + + self._update_url_map() + self._setattr_url_map() + + def get_conf(self): + ''' + Combine the CherryPy configuration with the rest_cherrypy config values + pulled from the master config and return the CherryPy configuration + ''' + conf = { + 'global': { + 'server.socket_host': self.apiopts.get('host', '0.0.0.0'), + 'server.socket_port': self.apiopts.get('port', 8000), + 'server.thread_pool': self.apiopts.get('thread_pool', 100), + 'server.socket_queue_size': self.apiopts.get('queue_size', 30), + 'max_request_body_size': self.apiopts.get('max_request_body_size', 1048576), + 'debug': self.apiopts.get('debug', False), + }, + '/': { + 'request.dispatch': cherrypy.dispatch.MethodDispatcher(), + + 'tools.trailing_slash.on': True, + 'tools.gzip.on': True, + + 'tools.cpstats.on': self.apiopts.get('collect_stats', False), + }, + } + + if self.apiopts.get('debug', False) == False: + conf['global']['environment'] = 'production' + + # Serve static media if the directory has been set in the configuration + if 'static' in self.apiopts: + conf[self.apiopts.get('static_path', '/static')] = { + 'tools.staticdir.on': True, + 'tools.staticdir.dir': self.apiopts['static'], + } + + # Add to global config + cherrypy.config.update(conf['global']) + + return conf + + +def get_app(opts): + ''' + Returns a WSGI app and a configuration dictionary + ''' + apiopts = opts.get(__name__.rsplit('.', 2)[-2], {}) # rest_cherrypy opts + + # Add Salt and salt-api config options to the main CherryPy config dict + cherrypy.config['saltopts'] = opts + cherrypy.config['apiopts'] = apiopts + + root = API() # cherrypy app + cpyopts = root.get_conf() # cherrypy app opts + + return root, apiopts, cpyopts diff --git a/salt/netapi/rest_cherrypy/event_processor.py b/salt/netapi/rest_cherrypy/event_processor.py new file mode 100644 index 00000000000..46d1c5b7b40 --- /dev/null +++ b/salt/netapi/rest_cherrypy/event_processor.py @@ -0,0 +1,206 @@ +import json +import logging + +logger = logging.getLogger(__name__) + + +class SaltInfo: + ''' + Class to handle processing and publishing of "real time" Salt upates. + ''' + + def __init__(self, handler): + ''' + handler is expected to be the server side end of a websocket + connection. + ''' + self.handler = handler + + ''' + These represent a "real time" view into Salt's jobs. + ''' + self.jobs = {} + + ''' + This represents a "real time" view of minions connected to + Salt. + ''' + self.minions = {} + + def publish_minions(self): + ''' + Publishes minions as a list of dicts. + ''' + minions = [] + + for minion, minion_info in self.minions.iteritems(): + curr_minion = {} + curr_minion.update(minion_info) + curr_minion.update({'id': minion}) + minions.append(curr_minion) + + ret = {'minions': minions} + self.handler.send(json.dumps(ret), False) + + def publish(self, key, data): + ''' + Publishes the data to the event stream. + ''' + publish_data = {key: data} + self.handler.send(json.dumps(publish_data), False) + + def process_minion_update(self, event_data): + ''' + Associate grains data with a minion and publish minion update + ''' + tag = event_data['tag'] + event_info = event_data['data'] + + _, _, _, _, mid = tag.split('/') + + if not self.minions.get(mid, None): + self.minions[mid] = {} + + minion = self.minions[mid] + + minion.update({'grains': event_info['return']}) + + self.publish_minions() + + def process_ret_job_event(self, event_data): + ''' + Process a /ret event returned by Salt for a particular minion. + These events contain the returned results from a particular execution. + ''' + tag = event_data['tag'] + event_info = event_data['data'] + + _, _, jid, _, mid = tag.split('/') + job = self.jobs.setdefault(jid, {}) + + minion = job.setdefault('minions', {}).setdefault(mid, {}) + minion.update({'return': event_info['return']}) + minion.update({'retcode': event_info['retcode']}) + minion.update({'success': event_info['success']}) + + job_complete = all([minion['success'] for mid, minion + in job['minions'].iteritems()]) + + if job_complete: + job['state'] = 'complete' + + self.publish('jobs', self.jobs) + + def process_new_job_event(self, event_data): + ''' + Creates a new job with properties from the event data + like jid, function, args, timestamp. + + Also sets the initial state to started. + + Minions that are participating in this job are also noted. + + ''' + job = None + tag = event_data['tag'] + event_info = event_data['data'] + minions = {} + for mid in event_info['minions']: + minions[mid] = {'success': False} + + job = { + 'jid': event_info['jid'], + 'start_time': event_info['_stamp'], + 'minions': minions, # is a dictionary keyed by mids + 'fun': event_info['fun'], + 'tgt': event_info['tgt'], + 'tgt_type': event_info['tgt_type'], + 'state': 'running', + } + self.jobs[event_info['jid']] = job + self.publish('jobs', self.jobs) + + def process_key_event(self, event_data): + tag = event_data['tag'] + event_info = event_data['data'] + + ''' + Tag: salt/key + Data: + {'_stamp': '2014-05-20T22:45:04.345583', + 'act': 'delete', + 'id': 'compute.home', + 'result': True} + ''' + + if event_info['act'] == 'delete': + self.minions.pop(event_info['id'], None) + elif event_info['act'] == 'accept': + self.minions.setdefault(event_info['id'], {}) + + self.publish_minions() + + def process_presense_events(salt_data, token, opts): + ''' + Check if any minions have connected or dropped. + Send a message to the client if they have. + ''' + tag = event_data['tag'] + event_info = event_data['data'] + + minions_detected = event_info['present'] + curr_minions = self.minions.keys() + + changed = False + + # check if any connections were dropped + dropped_minions = set(curr_minions) - set(minions_detected) + + for minion in dropped_minions: + changed = True + self.minions.pop(minion, None) + + # check if any new connections were made + new_minions = set(minions_detected) - set(curr_minions) + + tgt = ','.join(new_minions) + + if tgt: + changed = True + client = salt.netapi.NetapiClient(opts) + client.run( + { + 'fun': 'grains.items', + 'tgt': tgt, + 'expr_type': 'list', + 'mode': 'client', + 'client': 'local', + 'async': 'local_async', + 'token': token, + }) + + if changed: + self.publish_minions() + + def process(self, salt_data, token, opts): + ''' + Process events and publish data + ''' + parts = salt_data['tag'].split('/') + if len(parts) < 2: + return + + # TBD: Simplify these conditional expressions + if parts[1] == 'job': + if parts[3] == 'new': + self.process_new_job_event(salt_data) + if salt_data['data']['fun'] == 'grains.items': + self.minions = {} + elif parts[3] == 'ret': + self.process_ret_job_event(salt_data) + if salt_data['data']['fun'] == 'grains.items': + self.process_minion_update(salt_data) + if parts[1] == 'key': + self.process_key_event(salt_data) + if parts[1] == 'presense': + self.process_presense_events(salt_data, token, opts) diff --git a/salt/netapi/rest_cherrypy/tools/__init__.py b/salt/netapi/rest_cherrypy/tools/__init__.py new file mode 100644 index 00000000000..5ecc16c6106 --- /dev/null +++ b/salt/netapi/rest_cherrypy/tools/__init__.py @@ -0,0 +1,3 @@ +from .rate_limit import RateLimitTool + +__all__ = ('RateLimitTool',) diff --git a/salt/netapi/rest_cherrypy/tools/websockets.py b/salt/netapi/rest_cherrypy/tools/websockets.py new file mode 100644 index 00000000000..0f15f83353a --- /dev/null +++ b/salt/netapi/rest_cherrypy/tools/websockets.py @@ -0,0 +1,60 @@ +import cherrypy + +from ws4py.server.cherrypyserver import WebSocketPlugin, WebSocketTool +from ws4py.websocket import WebSocket +from multiprocessing import Lock, Pipe + +cherrypy.tools.websocket = WebSocketTool() +WebSocketPlugin(cherrypy.engine).subscribe() + + +class SynchronizingWebsocket(WebSocket): + ''' + Class to handle requests sent to this websocket connection. + Each instance of this class represents a Salt websocket connection. + Waits to receive a ``ready`` message fom the client. + Calls send on it's end of the pipe to signal to the sender on receipt + of ``ready``. + + This class also kicks off initial information probing jobs when clients + initially connect. These jobs help gather information about minions, jobs, + and documentation. + ''' + def __init__(self, *args, **kwargs): + super(SynchronizingWebsocket, self).__init__(*args, **kwargs) + + ''' + This pipe needs to represent the parent end of a pipe. + Clients need to ensure that the pipe assigned to ``self.pipe`` is + the ``parent end`` of a + `pipe `_. + ''' + self.pipe = None + + ''' + The token that we can use to make API calls. + There are times when we would like to kick off jobs, + examples include trying to obtain minions connected. + ''' + self.token = None + + ''' + Options represent ``salt`` options defined in the configs. + ''' + self.opts = None + + def received_message(self, message): + ''' + Checks if the client has sent a ready message. + A ready message causes ``send()`` to be called on the + ``parent end`` of the pipe. + + Clients need to ensure that the pipe assigned to ``self.pipe`` is + the ``parent end`` of a pipe. + + This ensures completion of the underlying websocket connection + and can be used to synchronize parallel senders. + ''' + if message.data == 'websocket client ready': + self.pipe.send(message) + self.send('server received message', False) diff --git a/salt/netapi/rest_cherrypy/wsgi.py b/salt/netapi/rest_cherrypy/wsgi.py new file mode 100644 index 00000000000..c02c319782a --- /dev/null +++ b/salt/netapi/rest_cherrypy/wsgi.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +''' +Deployment +========== + +The ``rest_cherrypy`` netapi module is a standard Python WSGI app. It can be +deployed one of two ways. + +:program:`salt-api` using the CherryPy server +--------------------------------------------- + +The default configuration is to run this module using :program:`salt-api` to +start the Python-based CherryPy server. This server is lightweight, +multi-threaded, encrypted with SSL, and should be considered production-ready. + +Using a WSGI-compliant web server +--------------------------------- + +This module may be deplayed on any WSGI-compliant server such as Apache with +mod_wsgi or Nginx with FastCGI, to name just two (there are many). + +Note, external WSGI servers handle URLs, paths, and SSL certs directly. The +``rest_cherrypy`` configuration options are ignored and the ``salt-api`` daemon +does not need to be running at all. Remember Salt authentication credentials +are sent in the clear unless SSL is being enforced! + +An example Apache virtual host configuration:: + + + ServerName example.com + ServerAlias *.example.com + + ServerAdmin webmaster@example.com + + LogLevel warn + ErrorLog /var/www/example.com/logs/error.log + CustomLog /var/www/example.com/logs/access.log combined + + DocumentRoot /var/www/example.com/htdocs + + WSGIScriptAlias / /path/to/salt/netapi/rest_cherrypy/wsgi.py + + +''' +# pylint: disable=C0103 + +import os + +import cherrypy + +def bootstrap_app(): + ''' + Grab the opts dict of the master config by trying to import Salt + ''' + from salt.netapi.rest_cherrypy import app + import salt.config + + __opts__ = salt.config.client_config( + os.environ.get('SALT_MASTER_CONFIG', '/etc/salt/master')) + return app.get_app(__opts__) + + +def get_application(*args): + ''' + Returns a WSGI application function. If you supply the WSGI app and config + it will use that, otherwise it will try to obtain them from a local Salt + installation + ''' + opts_tuple = args + + def wsgi_app(environ, start_response): + root, _, conf = opts_tuple or bootstrap_app() + cherrypy.config.update({'environment': 'embedded'}) + + cherrypy.tree.mount(root, '/', conf) + return cherrypy.tree(environ, start_response) + + return wsgi_app + +application = get_application() diff --git a/salt/netapi/rest_tornado/__init__.py b/salt/netapi/rest_tornado/__init__.py new file mode 100644 index 00000000000..d4d359553dc --- /dev/null +++ b/salt/netapi/rest_tornado/__init__.py @@ -0,0 +1,102 @@ +import hashlib +import logging + +__virtualname__ = 'rest_tornado' + +logger = logging.getLogger(__virtualname__) + +try: + import tornado.httpserver + import tornado.ioloop + import tornado.web + import tornado.gen + + has_tornado = True +except ImportError as err: + has_tornado = False + logger.info('ImportError! {}'.format(str(err))) + +import salt.auth + + + +def __virtual__(): + mod_opts = __opts__.get(__virtualname__, {}) + + if has_tornado and 'port' in mod_opts: + return __virtualname__ + + return False + + +def start(): + ''' + Start the saltnado! + ''' + from . import saltnado + + mod_opts = __opts__.get(__virtualname__, {}) + + if 'num_processes' not in mod_opts: + mod_opts['num_processes'] = 1 + + token_pattern = r"([0-9A-Fa-f]{%s})" % len(getattr(hashlib, __opts__.get('hash_type', 'md5'))().hexdigest()) + + all_events_pattern = r"/all_events/{}".format(token_pattern) + formatted_events_pattern = r"/formatted_events/{}".format(token_pattern) + logger.debug("All events URL pattern is {}".format(all_events_pattern)) + + application = tornado.web.Application([ + (r"/", saltnado.SaltAPIHandler), + (r"/login", saltnado.SaltAuthHandler), + (r"/minions/(.*)", saltnado.MinionSaltAPIHandler), + (r"/minions", saltnado.MinionSaltAPIHandler), + (r"/jobs/(.*)", saltnado.JobsSaltAPIHandler), + (r"/jobs", saltnado.JobsSaltAPIHandler), + (r"/run", saltnado.RunSaltAPIHandler), + (r"/events", saltnado.EventsSaltAPIHandler), + (r"/hook(/.*)?", saltnado.WebhookSaltAPIHandler), + # Matches /all_events/[0-9A-Fa-f]{n} + # Where n is the length of hexdigest + # for the current hashing algorithm. + # This algorithm is specified in the + # salt master config file. + (all_events_pattern, saltnado.AllEventsHandler), + (formatted_events_pattern, saltnado.FormattedEventsHandler), + ], debug=mod_opts.get('debug', False)) + + application.opts = __opts__ + application.mod_opts = mod_opts + application.auth = salt.auth.LoadAuth(__opts__) + application.event_listener = saltnado.EventListener(mod_opts, __opts__) + + # the kwargs for the HTTPServer + kwargs = {} + if not mod_opts.get('disable_ssl', False): + if 'ssl_crt' not in mod_opts: + logger.error("Not starting '%s'. Options 'ssl_crt' and " + "'ssl_key' are required if SSL is not disabled.", + __name__) + + return None + # cert is required, key may be optional + # https://docs.python.org/2/library/ssl.html#ssl.wrap_socket + ssl_opts = {'certfile': mod_opts['ssl_crt']} + if mod_opts.get('ssl_key', False): + ssl_opts.update({'keyfile': mod_opts['ssl_key']}) + kwargs['ssl_options'] = ssl_opts + + http_server = tornado.httpserver.HTTPServer(application, **kwargs) + try: + http_server.bind(mod_opts['port']) + http_server.start(mod_opts['num_processes']) + except: + print 'Rest_tornado unable to bind to port {0}'.format(mod_opts['port']) + raise SystemExit(1) + tornado.ioloop.IOLoop.instance().add_callback(application.event_listener.iter_events) + + + try: + tornado.ioloop.IOLoop.instance().start() + except KeyboardInterrupt: + raise SystemExit(0) diff --git a/salt/netapi/rest_tornado/event_processor.py b/salt/netapi/rest_tornado/event_processor.py new file mode 100644 index 00000000000..b7c6b73b1f3 --- /dev/null +++ b/salt/netapi/rest_tornado/event_processor.py @@ -0,0 +1,234 @@ +import json +import logging + +import salt.netapi + +logger = logging.getLogger(__name__) + + +class SaltInfo: + ''' + Class to handle processing and publishing of "real time" Salt upates. + ''' + + def __init__(self, handler): + ''' + handler is expected to be the server side end of a websocket + connection. + ''' + self.handler = handler + + ''' + These represent a "real time" view into Salt's jobs. + ''' + self.jobs = {} + + ''' + This represents a "real time" view of minions connected to + Salt. + ''' + self.minions = {} + + def publish_minions(self): + ''' + Publishes minions as a list of dicts. + ''' + logger.debug('in publish minions') + minions = {} + + logger.debug('starting loop') + for minion, minion_info in self.minions.iteritems(): + logger.debug(minion) + # logger.debug(minion_info) + curr_minion = {} + curr_minion.update(minion_info) + curr_minion.update({'id': minion}) + minions[minion] = curr_minion + logger.debug('ended loop') + ret = {'minions': minions} + self.handler.write_message(u'{}\n\n'.format(json.dumps(ret))) + + def publish(self, key, data): + ''' + Publishes the data to the event stream. + ''' + publish_data = {key: data} + pub = u'{}\n\n'.format(json.dumps(publish_data)) + self.handler.write_message(pub) + + def process_minion_update(self, event_data): + ''' + Associate grains data with a minion and publish minion update + ''' + tag = event_data['tag'] + event_info = event_data['data'] + + _, _, _, _, mid = tag.split('/') + + if not self.minions.get(mid, None): + self.minions[mid] = {} + + minion = self.minions[mid] + + minion.update({'grains': event_info['return']}) + logger.debug("In process minion grains update with minions={}".format(self.minions.keys())) + self.publish_minions() + + def process_ret_job_event(self, event_data): + ''' + Process a /ret event returned by Salt for a particular minion. + These events contain the returned results from a particular execution. + ''' + tag = event_data['tag'] + event_info = event_data['data'] + + _, _, jid, _, mid = tag.split('/') + job = self.jobs.setdefault(jid, {}) + + minion = job.setdefault('minions', {}).setdefault(mid, {}) + minion.update({'return': event_info['return']}) + minion.update({'retcode': event_info['retcode']}) + minion.update({'success': event_info['success']}) + + job_complete = all([minion['success'] for mid, minion + in job['minions'].iteritems()]) + + if job_complete: + job['state'] = 'complete' + + self.publish('jobs', self.jobs) + + def process_new_job_event(self, event_data): + ''' + Creates a new job with properties from the event data + like jid, function, args, timestamp. + + Also sets the initial state to started. + + Minions that are participating in this job are also noted. + + ''' + job = None + tag = event_data['tag'] + event_info = event_data['data'] + minions = {} + for mid in event_info['minions']: + minions[mid] = {'success': False} + + job = { + 'jid': event_info['jid'], + 'start_time': event_info['_stamp'], + 'minions': minions, # is a dictionary keyed by mids + 'fun': event_info['fun'], + 'tgt': event_info['tgt'], + 'tgt_type': event_info['tgt_type'], + 'state': 'running', + } + self.jobs[event_info['jid']] = job + self.publish('jobs', self.jobs) + + def process_key_event(self, event_data): + tag = event_data['tag'] + event_info = event_data['data'] + + ''' + Tag: salt/key + Data: + {'_stamp': '2014-05-20T22:45:04.345583', + 'act': 'delete', + 'id': 'compute.home', + 'result': True} + ''' + + if event_info['act'] == 'delete': + self.minions.pop(event_info['id'], None) + elif event_info['act'] == 'accept': + self.minions.setdefault(event_info['id'], {}) + + self.publish_minions() + + def process_presence_events(self, salt_data, token, opts): + ''' + Check if any minions have connected or dropped. + Send a message to the client if they have. + ''' + logger.debug('In presence') + changed = False + + # check if any connections were dropped + if set(salt_data['data'].get('lost', [])): + dropped_minions = set(salt_data['data'].get('lost', [])) + else: + dropped_minions = set(self.minions.keys()) - set(salt_data['data'].get('present', [])) + + for minion in dropped_minions: + changed = True + logger.debug('Popping {}'.format(minion)) + self.minions.pop(minion, None) + + # check if any new connections were made + if set(salt_data['data'].get('new', [])): + logger.debug('got new minions') + new_minions = set(salt_data['data'].get('new', [])) + changed = True + elif set(salt_data['data'].get('present', [])) - set(self.minions.keys()): + logger.debug('detected new minions') + new_minions = set(salt_data['data'].get('present', [])) - set(self.minions.keys()) + changed = True + else: + new_minions = [] + + tgt = ','.join(new_minions) + for mid in new_minions: + logger.debug('Adding minion') + self.minions[mid] = {} + + if tgt: + changed = True + client = salt.netapi.NetapiClient(opts) + client.run( + { + 'fun': 'grains.items', + 'tgt': tgt, + 'expr_type': 'list', + 'mode': 'client', + 'client': 'local', + 'async': 'local_async', + 'token': token, + }) + + if changed: + self.publish_minions() + + def process(self, salt_data, token, opts): + ''' + Process events and publish data + ''' + import threading + logger.debug('In process {}'.format(threading.current_thread())) + logger.debug(salt_data['tag']) + logger.debug(salt_data) + + parts = salt_data['tag'].split('/') + if len(parts) < 2: + return + + # TBD: Simplify these conditional expressions + if parts[1] == 'job': + logger.debug('In job part 1') + if parts[3] == 'new': + logger.debug('In new job') + self.process_new_job_event(salt_data) + # if salt_data['data']['fun'] == 'grains.items': + # self.minions = {} + elif parts[3] == 'ret': + logger.debug('In ret') + self.process_ret_job_event(salt_data) + if salt_data['data']['fun'] == 'grains.items': + self.process_minion_update(salt_data) + elif parts[1] == 'key': + logger.debug('In key') + self.process_key_event(salt_data) + elif parts[1] == 'presence': + self.process_presence_events(salt_data, token, opts) + # logger.debug('In presence') diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py new file mode 100644 index 00000000000..99e9d4ca966 --- /dev/null +++ b/salt/netapi/rest_tornado/saltnado.py @@ -0,0 +1,1116 @@ +''' +A REST API for Salt +=================== + +.. py:currentmodule:: salt.netapi.rest_tornado.saltnado + +:depends: - tornado Python module + +All Events +---------- + +Exposes ``all`` "real-time" events from Salt's event bus on a websocket connection. +It should be noted that "Real-time" here means these events are made available +to the server as soon as any salt related action (changes to minions, new jobs etc) happens. +Clients are however assumed to be able to tolerate any network transport related latencies. +Functionality provided by this endpoint is similar to the ``/events`` end point. + +The event bus on the Salt master exposes a large variety of things, notably +when executions are started on the master and also when minions ultimately +return their results. This URL provides a real-time window into a running +Salt infrastructure. Uses websocket as the transport mechanism. + +Exposes GET method to return websocket connections. +All requests should include an auth token. +A way to obtain obtain authentication tokens is shown below. + +.. code-block:: bash + + % curl -si localhost:8000/login \\ + -H "Accept: application/json" \\ + -d username='salt' \\ + -d password='salt' \\ + -d eauth='pam' + +Which results in the response + +.. code-block:: json + + { + "return": [{ + "perms": [".*", "@runner", "@wheel"], + "start": 1400556492.277421, + "token": "d0ce6c1a37e99dcc0374392f272fe19c0090cca7", + "expire": 1400599692.277422, + "user": "salt", + "eauth": "pam" + }] + } + +In this example the ``token`` returned is ``d0ce6c1a37e99dcc0374392f272fe19c0090cca7`` and can be included +in subsequent websocket requests (as part of the URL). + +The event stream can be easily consumed via JavaScript: + +.. code-block:: javascript + + // Note, you must be authenticated! + + // Get the Websocket connection to Salt + var source = new Websocket('wss://localhost:8000/all_events/d0ce6c1a37e99dcc0374392f272fe19c0090cca7'); + + // Get Salt's "real time" event stream. + source.onopen = function() { source.send('websocket client ready'); }; + + // Other handlers + source.onerror = function(e) { console.debug('error!', e); }; + + // e.data represents Salt's "real time" event data as serialized JSON. + source.onmessage = function(e) { console.debug(e.data); }; + + // Terminates websocket connection and Salt's "real time" event stream on the server. + source.close(); + +Or via Python, using the Python module +`websocket-client `_ for example. +Or the tornado +`client `_. + +.. code-block:: python + + # Note, you must be authenticated! + + from websocket import create_connection + + # Get the Websocket connection to Salt + ws = create_connection('wss://localhost:8000/all_events/d0ce6c1a37e99dcc0374392f272fe19c0090cca7') + + # Get Salt's "real time" event stream. + ws.send('websocket client ready') + + + # Simple listener to print results of Salt's "real time" event stream. + # Look at https://pypi.python.org/pypi/websocket-client/ for more examples. + while listening_to_events: + print ws.recv() # Salt's "real time" event data as serialized JSON. + + # Terminates websocket connection and Salt's "real time" event stream on the server. + ws.close() + + # Please refer to https://github.com/liris/websocket-client/issues/81 when using a self signed cert + +Above examples show how to establish a websocket connection to Salt and activating +real time updates from Salt's event stream by signaling ``websocket client ready``. + + +Formatted Events +----------------- + +Exposes ``formatted`` "real-time" events from Salt's event bus on a websocket connection. +It should be noted that "Real-time" here means these events are made available +to the server as soon as any salt related action (changes to minions, new jobs etc) happens. +Clients are however assumed to be able to tolerate any network transport related latencies. +Functionality provided by this endpoint is similar to the ``/events`` end point. + +The event bus on the Salt master exposes a large variety of things, notably +when executions are started on the master and also when minions ultimately +return their results. This URL provides a real-time window into a running +Salt infrastructure. Uses websocket as the transport mechanism. + +Formatted events parses the raw "real time" event stream and maintains +a current view of the following: + +- minions +- jobs + +A change to the minions (such as addition, removal of keys or connection drops) +or jobs is processed and clients are updated. +Since we use salt's presence events to track minions, +please enable ``presence_events`` +and set a small value for the ``loop_interval`` +in the salt master config file. + +Exposes GET method to return websocket connections. +All requests should include an auth token. +A way to obtain obtain authentication tokens is shown below. + +.. code-block:: bash + + % curl -si localhost:8000/login \\ + -H "Accept: application/json" \\ + -d username='salt' \\ + -d password='salt' \\ + -d eauth='pam' + +Which results in the response + +.. code-block:: json + + { + "return": [{ + "perms": [".*", "@runner", "@wheel"], + "start": 1400556492.277421, + "token": "d0ce6c1a37e99dcc0374392f272fe19c0090cca7", + "expire": 1400599692.277422, + "user": "salt", + "eauth": "pam" + }] + } + +In this example the ``token`` returned is ``d0ce6c1a37e99dcc0374392f272fe19c0090cca7`` and can be included +in subsequent websocket requests (as part of the URL). + +The event stream can be easily consumed via JavaScript: + +.. code-block:: javascript + + // Note, you must be authenticated! + + // Get the Websocket connection to Salt + var source = new Websocket('wss://localhost:8000/formatted_events/d0ce6c1a37e99dcc0374392f272fe19c0090cca7'); + + // Get Salt's "real time" event stream. + source.onopen = function() { source.send('websocket client ready'); }; + + // Other handlers + source.onerror = function(e) { console.debug('error!', e); }; + + // e.data represents Salt's "real time" event data as serialized JSON. + source.onmessage = function(e) { console.debug(e.data); }; + + // Terminates websocket connection and Salt's "real time" event stream on the server. + source.close(); + +Or via Python, using the Python module +`websocket-client `_ for example. +Or the tornado +`client `_. + +.. code-block:: python + + # Note, you must be authenticated! + + from websocket import create_connection + + # Get the Websocket connection to Salt + ws = create_connection('wss://localhost:8000/formatted_events/d0ce6c1a37e99dcc0374392f272fe19c0090cca7') + + # Get Salt's "real time" event stream. + ws.send('websocket client ready') + + + # Simple listener to print results of Salt's "real time" event stream. + # Look at https://pypi.python.org/pypi/websocket-client/ for more examples. + while listening_to_events: + print ws.recv() # Salt's "real time" event data as serialized JSON. + + # Terminates websocket connection and Salt's "real time" event stream on the server. + ws.close() + + # Please refer to https://github.com/liris/websocket-client/issues/81 when using a self signed cert + +Above examples show how to establish a websocket connection to Salt and activating +real time updates from Salt's event stream by signaling ``websocket client ready``. + +Example responses +----------------- + +``Minion information`` is a dictionary keyed by each connected minion's ``id`` (``mid``), +grains information for each minion is also included. + +Minion information is sent in response to the following minion events: + +- connection drops + - requires running ``manage.present`` periodically every ``loop_interval`` seconds +- minion addition +- minon removal + +.. code-block:: python + + # Not all grains are shown + data: { + "minions": { + "minion1": { + "id": "minion1", + "grains": { + "kernel": "Darwin", + "domain": "local", + "zmqversion": "4.0.3", + "kernelrelease": "13.2.0" + } + } + } + } + +``Job information`` is also tracked and delivered. + +Job information is also a dictionary +in which each job's information is keyed by salt's ``jid``. + +.. code-block:: python + + data: { + "jobs": { + "20140609153646699137": { + "tgt_type": "glob", + "jid": "20140609153646699137", + "tgt": "*", + "start_time": "2014-06-09T15:36:46.700315", + "state": "complete", + "fun": "test.ping", + "minions": { + "minion1": { + "return": true, + "retcode": 0, + "success": true + } + } + } + } + } + +Setup +===== + +In order to run rest_tornado with the salt-master +add the following to your salt master config file. + +.. code-block:: yaml + + rest_tornado: + # can be any port + port: 8000 + ssl_crt: /etc/pki/api/certs/server.crt + # no need to specify ssl_key if cert and key + # are in one single file + ssl_key: /etc/pki/api/certs/server.key + debug: False + disable_ssl: False + +''' + + +''' +Notes +===== + +.. code-block:: bash + + curl localhost:8888/login -d client=local -d username=username -d password=password -d eauth=pam + + # for testing + curl -H 'X-Auth-Token: 89010c15bcbc8e4fc4ce4605b6699165' localhost:8888 -d client=local -d tgt='*' -d fun='test.ping' + + # not working.... but in siege 3.0.1 and posts.. + siege -c 1 -n 1 "http://127.0.0.1:8888 POST client=local&tgt=*&fun=test.ping" + + # this works + ab - c 50 -n 100 -p body -T 'application/x-www-form-urlencoded' http://localhost:8888/ + + {"return": [{"perms": ["*.*"], "start": 1396151398.373983, "token": "cb86b805e8915c84bceb0d466026caab", "expire": 1396194598.373983, "user": "jacksontj", "eauth": "pam"}]}[jacksontj@Thomas-PC netapi]$ +''' + +import logging +from copy import copy + +import time + +import sys + +import tornado.httpserver +import tornado.ioloop +import tornado.web +import tornado.gen +import tornado.websocket +from tornado.concurrent import Future +import event_processor + +from collections import defaultdict + +import math +import functools +import json +import yaml +import zmq +import fnmatch + +# salt imports +import salt.netapi +import salt.utils +import salt.utils.event +from salt.utils.event import tagify +import salt.client +import salt.runner +import salt.auth + +logger = logging.getLogger() + +''' +The clients rest_cherrypi supports. We want to mimic the interface, but not + necessarily use the same API under the hood +# all of these require coordinating minion stuff + - "local" (done) + - "local_async" (done) + - "local_batch" (done) + +# master side + - "runner" (done) + - "wheel" (need async api...) +''' + + +# TODO: refreshing clients using cachedict +saltclients = {'local': salt.client.get_local_client().run_job, + # not the actual client we'll use.. but its what we'll use to get args + 'local_batch': salt.client.get_local_client().cmd_batch, + 'local_async': salt.client.get_local_client().run_job, + 'runner': salt.runner.RunnerClient(salt.config.master_config('/etc/salt/master')).async, + } + + +AUTH_TOKEN_HEADER = 'X-Auth-Token' +AUTH_COOKIE_NAME = 'session_id' + + +class TimeoutException(Exception): + pass + + +class Any(Future): + ''' + Future that wraps other futures to "block" until one is done + ''' + def __init__(self, futures): + super(Any, self).__init__() + for future in futures: + future.add_done_callback(self.done_callback) + + def done_callback(self, future): + self.set_result(future) + + +class EventListener(): + def __init__(self, mod_opts, opts): + self.mod_opts = mod_opts + self.opts = opts + self.event = salt.utils.event.get_event( + 'master', + opts['sock_dir'], + opts['transport']) + + # tag -> list of futures + self.tag_map = defaultdict(list) + + # request_obj -> list of (tag, future) + self.request_map = defaultdict(list) + + def clean_timeout_futures(self, request): + ''' + Remove all futures that were waiting for request `request` since it is done waiting + ''' + if request not in self.request_map: + return + for tag, future in self.request_map[request]: + # TODO: log, this shouldn't happen... + if tag not in self.tag_map: + continue + # mark the future done + future.set_exception(TimeoutException()) + self.tag_map[tag].remove(future) + + # if that was the last of them, remove the key all together + if len(self.tag_map[tag]) == 0: + del self.tag_map[tag] + + def get_event(self, request, + tag='', + callback=None): + ''' + Get an event (async of course) return a future that will get it later + ''' + future = Future() + if callback is not None: + def handle_future(future): + response = future.result() + self.io_loop.add_callback(callback, response) + future.add_done_callback(handle_future) + # add this tag and future to the callbacks + self.tag_map[tag].append(future) + self.request_map[request].append((tag, future)) + + return future + + def iter_events(self): + ''' + Iterate over all events that could happen + ''' + try: + data = self.event.get_event_noblock() + # see if we have any futures that need this info: + for tag_prefix, futures in self.tag_map.items(): + if data['tag'].startswith(tag_prefix): + for future in futures: + if future.done(): + continue + future.set_result(data) + del self.tag_map[tag_prefix] + + # call yourself back! + tornado.ioloop.IOLoop.instance().add_callback(self.iter_events) + + except zmq.ZMQError as e: + # TODO: not sure what other errors we can get... + if e.errno != zmq.EAGAIN: + raise Exception() + # add callback in the future (to avoid spinning) + # TODO: configurable timeout + tornado.ioloop.IOLoop.instance().add_timeout(time.time() + 0.1, self.iter_events) + except: + logging.critical('Uncaught exception in the event_listener: {0}'.format(sys.exc_info())) + # TODO: configurable timeout + tornado.ioloop.IOLoop.instance().add_timeout(time.time() + 0.1, self.iter_events) + + +# TODO: move to a utils function within salt-- the batching stuff is a bit tied together +def get_batch_size(batch, num_minions): + ''' + Return the batch size that you should have + ''' + # figure out how many we can keep in flight + partition = lambda x: float(x) / 100.0 * num_minions + try: + if '%' in batch: + res = partition(float(batch.strip('%'))) + if res < 1: + return int(math.ceil(res)) + else: + return int(res) + else: + return int(batch) + except ValueError: + print(('Invalid batch data sent: {0}\nData must be in the form' + 'of %10, 10% or 3').format(batch)) + + +class BaseSaltAPIHandler(tornado.web.RequestHandler): + ct_out_map = ( + ('application/json', json.dumps), + ('application/x-yaml', functools.partial( + yaml.safe_dump, default_flow_style=False)), + ) + + def _verify_client(self, client): + ''' + Verify that the client is in fact one we have + ''' + if client not in saltclients: + self.set_status(400) + self.write('We don\'t serve your kind here') + self.finish() + + @property + def token(self): + ''' + The token used for the request + ''' + # find the token (cookie or headers) + if AUTH_TOKEN_HEADER in self.request.headers: + return self.request.headers[AUTH_TOKEN_HEADER] + else: + return self.get_cookie(AUTH_COOKIE_NAME) + + def _verify_auth(self): + ''' + Boolean wether the request is auth'd + ''' + + return self.token and bool(self.application.auth.get_tok(self.token)) + + def prepare(self): + ''' + Run before get/posts etc. Pre-flight checks: + - verify that we can speak back to them (compatible accept header) + ''' + # verify the content type + found = False + for content_type, dumper in self.ct_out_map: + if fnmatch.fnmatch(content_type, self.request.headers['Accept']): + found = True + break + + # better return message? + if not found: + self.send_error(406) + + self.content_type = content_type + self.dumper = dumper + + # do the common parts + self.start = time.time() + self.connected = True + + self.lowstate = self._get_lowstate() + + def timeout_futures(self): + ''' + timeout a session + ''' + # TODO: set a header or something??? so we know it was a timeout + self.application.event_listener.clean_timeout_futures(self) + + def on_finish(self): + ''' + When the job has been done, lets cleanup + ''' + # timeout all the futures + self.timeout_futures() + + def on_connection_close(self): + ''' + If the client disconnects, lets close out + ''' + self.finish() + + def serialize(self, data): + ''' + Serlialize the output based on the Accept header + ''' + self.set_header('Content-Type', self.content_type) + + return self.dumper(data) + + def _form_loader(self, _): + ''' + function to get the data from the urlencoded forms + ignore the data passed in and just get the args from wherever they are + ''' + data = {} + for key, val in self.request.arguments.iteritems(): + if len(val) == 1: + data[key] = val[0] + else: + data[key] = val + return data + + def deserialize(self, data): + ''' + Deserialize the data based on request content type headers + ''' + ct_in_map = { + 'application/x-www-form-urlencoded': self._form_loader, + 'application/json': json.loads, + 'application/x-yaml': functools.partial( + yaml.safe_load, default_flow_style=False), + 'text/yaml': functools.partial( + yaml.safe_load, default_flow_style=False), + # because people are terrible and dont mean what they say + 'text/plain': json.loads + } + + try: + if self.request.headers['Content-Type'] not in ct_in_map: + self.send_error(406) + return ct_in_map[self.request.headers['Content-Type']](data) + except KeyError: + return [] + + def _get_lowstate(self): + ''' + Format the incoming data into a lowstate object + ''' + data = self.deserialize(self.request.body) + self.raw_data = copy(data) + + if self.request.headers.get('Content-Type') == 'application/x-www-form-urlencoded': + if 'arg' in data and not isinstance(data['arg'], list): + data['arg'] = [data['arg']] + lowstate = [data] + else: + lowstate = data + return lowstate + + +class SaltAuthHandler(BaseSaltAPIHandler): + ''' + Handler for login resquests + ''' + def get(self): + ''' + We don't allow gets on the login path, so lets send back a nice message + ''' + self.set_status(401) + self.set_header('WWW-Authenticate', 'Session') + + ret = {'status': '401 Unauthorized', + 'return': 'Please log in'} + + self.write(self.serialize(ret)) + self.finish() + + # TODO: make async? Underlying library isn't... and we ARE making disk calls :( + def post(self): + ''' + Authenticate against Salt's eauth system + {"return": {"start": 1395507384.320007, "token": "6ff4cd2b770ada48713afc629cd3178c", "expire": 1395550584.320007, "name": "jacksontj", "eauth": "pam"}} + {"return": [{"perms": ["*.*"], "start": 1395507675.396021, "token": "dea8274dc359fee86357d9d0263ec93c0498888e", "expire": 1395550875.396021, "user": "jacksontj", "eauth": "pam"}]} + ''' + creds = {'username': self.get_arguments('username')[0], + 'password': self.get_arguments('password')[0], + 'eauth': self.get_arguments('eauth')[0], + } + + token = self.application.auth.mk_token(creds) + if not 'token' in token: + # TODO: nicer error message + # 'Could not authenticate using provided credentials') + self.send_error(401) + # return since we don't want to execute any more + return + + # Grab eauth config for the current backend for the current user + try: + perms = self.application.opts['external_auth'][token['eauth']][token['name']] + except (AttributeError, IndexError): + logging.debug("Configuration for external_auth malformed for " + "eauth '{0}', and user '{1}'." + .format(token.get('eauth'), token.get('name')), exc_info=True) + # TODO better error -- 'Configuration for external_auth could not be read.' + self.send_error(500) + + ret = {'return': [{ + 'token': token['token'], + 'expire': token['expire'], + 'start': token['start'], + 'user': token['name'], + 'eauth': token['eauth'], + 'perms': perms, + }]} + + self.write(self.serialize(ret)) + self.finish() + + +class SaltAPIHandler(BaseSaltAPIHandler): + ''' + Main API handler for base "/" + ''' + def get(self): + ''' + return data about what clients you have + ''' + ret = {"clients": saltclients.keys(), + "return": "Welcome"} + self.write(self.serialize(ret)) + self.finish() + + @tornado.web.asynchronous + def post(self): + ''' + This function takes in all the args for dispatching requests + **Example request**:: + + % curl -si https://localhost:8000 \\ + -H "Accept: application/x-yaml" \\ + -H "X-Auth-Token: d40d1e1e" \\ + -d client=local \\ + -d tgt='*' \\ + -d fun='test.sleep' \\ + -d arg=1 + ''' + # if you aren't authenticated, redirect to login + if not self._verify_auth(): + self.redirect('/login') + return + + client = self.get_arguments('client')[0] + self._verify_client(client) + self.disbatch(client) + + def disbatch(self, client): + ''' + Disbatch a lowstate job to the appropriate client + ''' + self.client = client + + for low in self.lowstate: + if (not self._verify_auth() or 'eauth' in low): + # TODO: better error? + self.set_status(401) + self.finish() + return + # disbatch to the correct handler + try: + getattr(self, '_disbatch_{0}'.format(self.client))() + except AttributeError: + # TODO set the right status... this means we didn't implement it... + self.set_status(500) + self.finish() + + @tornado.gen.coroutine + def _disbatch_local_batch(self): + ''' + Disbatch local client batched commands + ''' + self.ret = [] + + for chunk in self.lowstate: + f_call = salt.utils.format_call(saltclients['local_batch'], chunk) + + timeout = float(chunk.get('timeout', self.application.opts['timeout'])) + # set the timeout + timeout_obj = tornado.ioloop.IOLoop.instance().add_timeout(time.time() + timeout, self.timeout_futures) + + # ping all the minions (to see who we have to talk to) + # TODO: actually ping them all? this just gets the pub data + minions = saltclients['local'](chunk['tgt'], + 'test.ping', + [], + expr_form=f_call['kwargs']['expr_form'])['minions'] + + chunk_ret = {} + maxflight = get_batch_size(f_call['kwargs']['batch'], len(minions)) + inflight_futures = [] + # do this batch + while len(minions) > 0: + # if you have more to go, lets disbatch jobs + while len(inflight_futures) < maxflight: + minion_id = minions.pop(0) + f_call['args'][0] = minion_id + # TODO: list?? + f_call['kwargs']['expr_form'] = 'glob' + pub_data = saltclients['local'](*f_call.get('args', ()), **f_call.get('kwargs', {})) + print pub_data + tag = tagify([pub_data['jid'], 'ret', minion_id], 'job') + future = self.application.event_listener.get_event(self, tag=tag) + inflight_futures.append(future) + + # wait until someone is done + finished_future = yield Any(inflight_futures) + try: + event = finished_future.result() + except TimeoutException: + break + print event + chunk_ret[event['data']['id']] = event['data']['return'] + inflight_futures.remove(finished_future) + + self.ret.append(chunk_ret) + + # if we finish in time, cancel the timeout + tornado.ioloop.IOLoop.instance().remove_timeout(timeout_obj) + + self.write(self.serialize({'return': self.ret})) + self.finish() + + @tornado.gen.coroutine + def _disbatch_local(self): + ''' + Disbatch local client commands + ''' + self.ret = [] + + for chunk in self.lowstate: + timeout = float(chunk.get('timeout', self.application.opts['timeout'])) + # set the timeout + tornado.ioloop.IOLoop.instance().add_timeout(time.time() + timeout, self.timeout_futures) + timeout_obj = tornado.ioloop.IOLoop.instance().add_timeout(time.time() + timeout, self.timeout_futures) + + # TODO: not sure why.... we already verify auth, probably for ACLs + # require token or eauth + chunk['token'] = self.token + + chunk_ret = {} + + f_call = salt.utils.format_call(saltclients[self.client], chunk) + # fire a job off + pub_data = saltclients[self.client](*f_call.get('args', ()), **f_call.get('kwargs', {})) + + # get the tag that we are looking for + tag = tagify([pub_data['jid'], 'ret'], 'job') + + minions_remaining = pub_data['minions'] + + # while we are waiting on all the mininons + while len(minions_remaining) > 0: + try: + event = yield self.application.event_listener.get_event(self, tag=tag) + chunk_ret[event['data']['id']] = event['data']['return'] + minions_remaining.remove(event['data']['id']) + # if you hit a timeout, just stop waiting ;) + except TimeoutException: + break + self.ret.append(chunk_ret) + + # if we finish in time, cancel the timeout + tornado.ioloop.IOLoop.instance().remove_timeout(timeout_obj) + + self.write(self.serialize({'return': self.ret})) + self.finish() + + def _disbatch_local_async(self): + ''' + Disbatch local client_async commands + ''' + ret = [] + for chunk in self.lowstate: + f_call = salt.utils.format_call(saltclients[self.client], chunk) + # fire a job off + pub_data = saltclients[self.client](*f_call.get('args', ()), **f_call.get('kwargs', {})) + ret.append(pub_data) + + self.write(self.serialize({'return': ret})) + self.finish() + + @tornado.gen.coroutine + def _disbatch_runner(self): + ''' + Disbatch runner client commands + ''' + self.ret = [] + for chunk in self.lowstate: + timeout = float(chunk.get('timeout', self.application.opts['timeout'])) + # set the timeout + tornado.ioloop.IOLoop.instance().add_timeout(time.time() + timeout, self.timeout_futures) + timeout_obj = tornado.ioloop.IOLoop.instance().add_timeout(time.time() + timeout, self.timeout_futures) + + f_call = {'args': [chunk['fun'], chunk]} + pub_data = saltclients[self.client](chunk['fun'], chunk) + tag = pub_data['tag'] + '/ret' + try: + event = yield self.application.event_listener.get_event(self, tag=tag) + # only return the return data + self.ret.append(event['data']['return']) + + # if we finish in time, cancel the timeout + tornado.ioloop.IOLoop.instance().remove_timeout(timeout_obj) + except TimeoutException: + break + + self.write(self.serialize({'return': self.ret})) + self.finish() + + +class MinionSaltAPIHandler(SaltAPIHandler): + ''' + Handler for /minion requests + ''' + @tornado.web.asynchronous + def get(self, mid): + # if you aren't authenticated, redirect to login + if not self._verify_auth(): + self.redirect('/login') + return + + #'client': 'local', 'tgt': mid or '*', 'fun': 'grains.items', + self.lowstate = [{ + 'client': 'local', 'tgt': mid or '*', 'fun': 'grains.items', + }] + self.disbatch('local') + + @tornado.web.asynchronous + def post(self): + ''' + local_async post endpoint + ''' + # if you aren't authenticated, redirect to login + if not self._verify_auth(): + self.redirect('/login') + return + + self.disbatch('local_async') + + +class JobsSaltAPIHandler(SaltAPIHandler): + ''' + Handler for /minion requests + ''' + @tornado.web.asynchronous + def get(self, jid=None): + # if you aren't authenticated, redirect to login + if not self._verify_auth(): + self.redirect('/login') + return + + self.lowstate = [{ + 'fun': 'jobs.lookup_jid' if jid else 'jobs.list_jobs', + 'jid': jid, + }] + + if jid: + self.lowstate.append({ + 'fun': 'jobs.list_job', + 'jid': jid, + }) + + self.disbatch('runner') + + +class RunSaltAPIHandler(SaltAPIHandler): + ''' + Handler for /run requests + ''' + @tornado.web.asynchronous + def post(self): + client = self.get_arguments('client')[0] + self._verify_client(client) + self.disbatch(client) + + +class EventsSaltAPIHandler(SaltAPIHandler): + ''' + Handler for /events requests + ''' + @tornado.gen.coroutine + def get(self): + # if you aren't authenticated, redirect to login + if not self._verify_auth(): + self.redirect('/login') + return + # set the streaming headers + self.set_header('Content-Type', 'text/event-stream') + self.set_header('Cache-Control', 'no-cache') + self.set_header('Connection', 'keep-alive') + + self.write(u'retry: {0}\n'.format(400)) + self.flush() + + while True: + try: + event = yield self.application.event_listener.get_event(self) + self.write(u'tag: {0}\n'.format(event.get('tag', ''))) + self.write(u'data: {0}\n\n'.format(json.dumps(event))) + self.flush() + except TimeoutException: + break + + self.finish() + + +class AllEventsHandler(tornado.websocket.WebSocketHandler): + ''' + Server side websocket handler. + ''' + def open(self, token): + ''' + Return a websocket connection to Salt + representing Salt's "real time" event stream. + ''' + logger.debug('In the websocket open method') + + self.token = token + # close the connection, if not authenticated + if not self.application.auth.get_tok(token): + logger.debug('Refusing websocket connection, bad token!') + self.close() + return + + self.connected = False + + @tornado.gen.coroutine + def on_message(self, message): + """Listens for a "websocket client ready" message. + Once that message is received an asynchronous job + is stated that yeilds messages to the client. + These messages make up salt's + "real time" event stream. + """ + logger.debug('Got websocket message {}'.format(message)) + if message == 'websocket client ready': + if self.connected: + # TBD: Add ability to run commands in this branch + logger.debug('Websocket already connected, returning') + return + + self.connected = True + + while True: + try: + event = yield self.application.event_listener.get_event(self) + self.write_message(u'data: {0}\n\n'.format(json.dumps(event))) + except Exception as err: + logger.info('Error! Ending server side websocket connection. Reason = {}'.format(str(err))) + break + + self.close() + else: + # TBD: Add logic to run salt commands here + pass + + def on_close(self, *args, **kwargs): + '''Cleanup. + + ''' + logger.debug('In the websocket close method') + self.close() + + +class FormattedEventsHandler(AllEventsHandler): + + @tornado.gen.coroutine + def on_message(self, message): + """Listens for a "websocket client ready" message. + Once that message is received an asynchronous job + is stated that yeilds messages to the client. + These messages make up salt's + "real time" event stream. + """ + logger.debug('Got websocket message {}'.format(message)) + if message == 'websocket client ready': + if self.connected: + # TBD: Add ability to run commands in this branch + logger.debug('Websocket already connected, returning') + return + + self.connected = True + + evt_processor = event_processor.SaltInfo(self) + client = salt.netapi.NetapiClient(self.application.opts) + client.run({ + 'fun': 'grains.items', + 'tgt': '*', + 'token': self.token, + 'mode': 'client', + 'async': 'local_async', + 'client': 'local' + }) + while True: + try: + event = yield self.application.event_listener.get_event(self) + evt_processor.process(event, self.token, self.application.opts) + # self.write_message(u'data: {0}\n\n'.format(json.dumps(event))) + except Exception as err: + logger.debug('Error! Ending server side websocket connection. Reason = {}'.format(str(err))) + break + + self.close() + else: + # TBD: Add logic to run salt commands here + pass + + +class WebhookSaltAPIHandler(SaltAPIHandler): + ''' + Handler for /run requests + ''' + def post(self, tag_suffix=None): + if not self._verify_auth(): + self.redirect('/login') + return + + # if you have the tag, prefix + tag = 'salt/netapi/hook' + if tag_suffix: + tag += tag_suffix + + # TODO: consolidate?? + self.event = salt.utils.event.get_event( + 'master', + self.application.opts['sock_dir'], + self.application.opts['transport']) + + ret = self.event.fire_event({ + 'post': self.raw_data, + 'headers': self.request.headers, + }, tag) + + self.write(self.serialize({'success': ret})) diff --git a/salt/netapi/rest_wsgi.py b/salt/netapi/rest_wsgi.py new file mode 100644 index 00000000000..bd8177cbcc3 --- /dev/null +++ b/salt/netapi/rest_wsgi.py @@ -0,0 +1,317 @@ +''' +A minimalist REST API for Salt +============================== + +This ``rest_wsgi`` module provides a no-frills REST interface for sending +commands to the Salt master. There are no dependencies. + +Extra care must be taken when deploying this module into production. Please +read this documentation in entirety. + +All authentication is done through Salt's :ref:`external auth ` +system. + +Usage +===== + +* All requests must be sent to the root URL (``/``). +* All requests must be sent as a POST request with JSON content in the request + body. +* All responses are in JSON. + +.. seealso:: :py:mod:`rest_cherrypy ` + + The :py:mod:`rest_cherrypy ` module is + more full-featured, production-ready, and has builtin security features. + +Deployment +========== + +The ``rest_wsgi`` netapi module is a standard Python WSGI app. It can be +deployed one of two ways. + +Using a WSGI-compliant web server +--------------------------------- + +This module may be run via any WSGI-compliant production server such as Apache +with mod_wsgi or Nginx with FastCGI. + +It is strongly recommended that this app be used with a server that supports +HTTPS encryption since raw Salt authentication credentials must be sent with +every request. Any apps that access Salt through this interface will need to +manually manage authentication credentials (either username and password or a +Salt token). Tread carefully. + +:program:`salt-api` using a development-only server +--------------------------------------------------- + +If run directly via the salt-api daemon it uses the `wsgiref.simple_server()`__ +that ships in the Python standard library. This is a single-threaded server +that is intended for testing and development. **This server does not use +encryption**; please note that raw Salt authentication credentials must be sent +with every HTTP request. + +**Running this module via salt-api is not recommended!** + +In order to start this module via the ``salt-api`` daemon the following must be +put into the Salt master config:: + + rest_wsgi: + port: 8001 + +.. __: http://docs.python.org/2/library/wsgiref.html#module-wsgiref.simple_server + +Usage examples +============== + +.. http:post:: / + + **Example request** for a basic ``test.ping``:: + + % curl -sS -i \\ + -H 'Content-Type: application/json' \\ + -d '[{"eauth":"pam","username":"saltdev","password":"saltdev","client":"local","tgt":"*","fun":"test.ping"}]' localhost:8001 + + **Example response**: + + .. code-block:: http + + HTTP/1.0 200 OK + Content-Length: 89 + Content-Type: application/json + + {"return": [{"ms--4": true, "ms--3": true, "ms--2": true, "ms--1": true, "ms--0": true}]} + + **Example request** for an asyncronous ``test.ping``:: + + % curl -sS -i \\ + -H 'Content-Type: application/json' \\ + -d '[{"eauth":"pam","username":"saltdev","password":"saltdev","client":"local_async","tgt":"*","fun":"test.ping"}]' localhost:8001 + + **Example response**: + + .. code-block:: http + + HTTP/1.0 200 OK + Content-Length: 103 + Content-Type: application/json + + {"return": [{"jid": "20130412192112593739", "minions": ["ms--4", "ms--3", "ms--2", "ms--1", "ms--0"]}]} + + **Example request** for looking up a job ID:: + + % curl -sS -i \\ + -H 'Content-Type: application/json' \\ + -d '[{"eauth":"pam","username":"saltdev","password":"saltdev","client":"runner","fun":"jobs.lookup_jid","jid":"20130412192112593739"}]' localhost:8001 + + **Example response**: + + .. code-block:: http + + HTTP/1.0 200 OK + Content-Length: 89 + Content-Type: application/json + + {"return": [{"ms--4": true, "ms--3": true, "ms--2": true, "ms--1": true, "ms--0": true}]} + +:form lowstate: A list of :term:`lowstate` data appropriate for the + :ref:`client ` interface you are calling. +:status 200: success +:status 401: authentication required + +''' +import errno +import json +import logging +import os + +# Import salt libs +import salt +import salt.netapi + +# HTTP response codes to response headers map +H = { + 200: '200 OK', + 400: '400 BAD REQUEST', + 401: '401 UNAUTHORIZED', + 404: '404 NOT FOUND', + 405: '405 METHOD NOT ALLOWED', + 406: '406 NOT ACCEPTABLE', + 500: '500 INTERNAL SERVER ERROR', +} + +__virtualname__ = 'rest_wsgi' + +logger = logging.getLogger(__virtualname__) + +def __virtual__(): + mod_opts = __opts__.get(__virtualname__, {}) + + if 'port' in mod_opts: + return __virtualname__ + + return False + +class HTTPError(Exception): + ''' + A custom exception that can take action based on an HTTP error code + ''' + def __init__(self, code, message): + self.code = code + Exception.__init__(self, '{0}: {1}'.format(code, message)) + +def mkdir_p(path): + ''' + mkdir -p + http://stackoverflow.com/a/600612/127816 + ''' + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: raise + +def read_body(environ): + ''' + Pull the body from the request and return it + ''' + length = environ.get('CONTENT_LENGTH', '0') + length = 0 if length == '' else int(length) + + return environ['wsgi.input'].read(length) + +def get_json(environ): + ''' + Return the request body as JSON + ''' + content_type = environ.get('CONTENT_TYPE', '') + if content_type != 'application/json': + raise HTTPError(406, 'JSON required') + + try: + return json.loads(read_body(environ)) + except ValueError as exc: + raise HTTPError(400, exc) + +def get_headers(data, extra_headers=None): + ''' + Takes the response data as well as any additional headers and returns a + tuple of tuples of headers suitable for passing to start_response() + ''' + response_headers = { + 'Content-Length': str(len(data)), + } + + if extra_headers: + response_headers.update(extra_headers) + + return response_headers.items() + +def run_chunk(environ, lowstate): + ''' + Expects a list of lowstate dictionaries that are executed and returned in + order + ''' + client = environ['SALT_APIClient'] + + for chunk in lowstate: + yield client.run(chunk) + +def dispatch(environ): + ''' + Do any path/method dispatching here and return a JSON-serializable data + structure appropriate for the response + ''' + method = environ['REQUEST_METHOD'].upper() + + if method == 'GET': + return ("They found me. I don't know how, but they found me. " + "Run for it, Marty!") + elif method == 'POST': + data = get_json(environ) + return run_chunk(environ, data) + else: + raise HTTPError(405, 'Method Not Allowed') + +def saltenviron(environ): + ''' + Make Salt's opts dict and the APIClient available in the WSGI environ + ''' + if not '__opts__' in locals(): + import salt.config + __opts__ = salt.config.client_config( + os.environ.get('SALT_MASTER_CONFIG', '/etc/salt/master')) + + environ['SALT_OPTS'] = __opts__ + environ['SALT_APIClient'] = salt.netapi.NetapiClient(__opts__) + +def application(environ, start_response): + ''' + Process the request and return a JSON response. Catch errors and return the + appropriate HTTP code. + ''' + # Instantiate APIClient once for the whole app + saltenviron(environ) + + # Call the dispatcher + try: + resp = list(dispatch(environ)) + code = 200 + except HTTPError as exc: + code = exc.code + resp = str(exc) + except salt.exceptions.EauthAuthenticationError as exc: + code = 401 + resp = str(exc) + except Exception as exc: + code = 500 + resp = str(exc) + + # Convert the response to JSON + try: + ret = json.dumps({'return': resp}) + except TypeError as exc: + code = 500 + ret = str(exc) + + # Return the response + start_response(H[code], get_headers(ret, { + 'Content-Type': 'application/json', + })) + return (ret,) + +def get_opts(): + ''' + Return the Salt master config as __opts__ + ''' + import salt.config + + return salt.config.client_config( + os.environ.get('SALT_MASTER_CONFIG', '/etc/salt/master')) + +def start(): + ''' + Start simple_server() + ''' + from wsgiref.simple_server import make_server + + # When started outside of salt-api __opts__ will not be injected + if not '__opts__' in globals(): + globals()['__opts__'] = get_opts() + + if __virtual__() == False: + raise SystemExit(1) + + mod_opts = __opts__.get(__virtualname__, {}) + + # pylint: disable-msg=C0103 + httpd = make_server('localhost', mod_opts['port'], application) + + try: + httpd.serve_forever() + except KeyboardInterrupt: + raise SystemExit(0) + +if __name__ == '__main__': + start() diff --git a/salt/scripts.py b/salt/scripts.py index 33ab3883e84..6327c6a1784 100644 --- a/salt/scripts.py +++ b/salt/scripts.py @@ -15,6 +15,7 @@ import logging import salt import salt.exceptions import salt.cli +import salt.cli.saltapi try: import salt.cloud.cli HAS_SALTCLOUD = True @@ -229,6 +230,14 @@ def salt_cloud(): hardcrash, trace=trace) +def salt_api(): + ''' + The main function for salt-api + ''' + sapi = salt.cli.saltapi.SaltAPI() + sapi.run() + + def salt_main(): ''' Publish commands to the salt system from the command line on the diff --git a/scripts/salt-api b/scripts/salt-api new file mode 100755 index 00000000000..b8d3168e632 --- /dev/null +++ b/scripts/salt-api @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +# Import salt libs +import salt.cli + +def main(): + sapi = salt.cli.SaltAPI() + sapi.run() + +if __name__ == '__main__': + main() diff --git a/tests/integration/netapi/__init__.py b/tests/integration/netapi/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/netapi/rest_cherrypy/__init__.py b/tests/integration/netapi/rest_cherrypy/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/netapi/rest_cherrypy/test_app.py b/tests/integration/netapi/rest_cherrypy/test_app.py new file mode 100644 index 00000000000..cd6e9af8d78 --- /dev/null +++ b/tests/integration/netapi/rest_cherrypy/test_app.py @@ -0,0 +1,101 @@ +# coding: utf-8 +import cgi +import json +import urllib + +import cherrypy +import yaml + +from tests.utils import BaseRestCherryPyTest + +class TestAuth(BaseRestCherryPyTest): + def test_get_root_noauth(self): + ''' + GET requests to the root URL should not require auth + ''' + request, response = self.request('/') + self.assertEqual(response.status, '200 OK') + + def test_post_root_auth(self): + ''' + POST requests to the root URL redirect to login + ''' + self.assertRaisesRegexp(cherrypy.InternalRedirect, '\/login', + self.request, '/', method='POST', data={}) + + def test_login_noauth(self): + ''' + GET requests to the login URL should not require auth + ''' + request, response = self.request('/login') + self.assertEqual(response.status, '200 OK') + + def test_webhook_auth(self): + ''' + Requests to the webhook URL require auth by default + ''' + self.assertRaisesRegexp(cherrypy.InternalRedirect, '\/login', + self.request, '/hook', method='POST', data={}) + +class TestLogin(BaseRestCherryPyTest): + auth_creds = ( + ('username', 'saltdev'), + ('password', 'saltdev'), + ('eauth', 'auto')) + + def test_good_login(self): + ''' + Test logging in + ''' + # Mock mk_token for a positive return + self.Resolver.return_value.mk_token.return_value = { + 'token': '6d1b722e', + 'start': 1363805943.776223, + 'expire': 1363849143.776224, + 'name': 'saltdev', + 'eauth': 'auto', + } + + body = urllib.urlencode(self.auth_creds) + request, response = self.request('/login', method='POST', body=body, + headers={ + 'content-type': 'application/x-www-form-urlencoded' + }) + self.assertEqual(response.status, '200 OK') + + def test_bad_login(self): + ''' + Test logging in + ''' + # Mock mk_token for a negative return + self.Resolver.return_value.mk_token.return_value = {} + + body = urllib.urlencode({'totally': 'invalid_creds'}) + request, response = self.request('/login', method='POST', body=body, + headers={ + 'content-type': 'application/x-www-form-urlencoded' + }) + self.assertEqual(response.status, '401 Unauthorized') + +class TestWebhookDisableAuth(BaseRestCherryPyTest): + __opts__ = { + 'rest_cherrypy': { + 'port': 8000, + 'debug': True, + 'webhook_disable_auth': True, + }, + } + + def test_webhook_noauth(self): + ''' + Auth can be disabled for requests to the webhook URL + ''' + # Mock fire_event() since we're only testing auth here. + self.get_event.return_value.fire_event.return_value = True + + body = urllib.urlencode({'foo': 'Foo!'}) + request, response = self.request('/hook', method='POST', body=body, + headers={ + 'content-type': 'application/x-www-form-urlencoded' + }) + self.assertEqual(response.status, '200 OK') diff --git a/tests/unit/netapi/__init__.py b/tests/unit/netapi/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/netapi/rest_cherrypy/__init__.py b/tests/unit/netapi/rest_cherrypy/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/netapi/rest_cherrypy/test_tools.py b/tests/unit/netapi/rest_cherrypy/test_tools.py new file mode 100644 index 00000000000..e5b578b4db9 --- /dev/null +++ b/tests/unit/netapi/rest_cherrypy/test_tools.py @@ -0,0 +1,81 @@ +# coding: utf-8 +import json +import urllib + +import cherrypy +import yaml + +from salt.netapi.rest_cherrypy import app + +from tests.utils import BaseRestCherryPyTest, BaseToolsTest + +class TestOutFormats(BaseToolsTest): + _cp_config = { + 'tools.hypermedia_out.on': True, + } + + def test_default_accept(self): + request, response = self.request('/') + self.assertEqual(response.headers['Content-type'], 'application/json') + + def test_unsupported_accept(self): + request, response = self.request('/', headers=( + ('Accept', 'application/ms-word'), + )) + self.assertEqual(response.status, '406 Not Acceptable') + + def test_json_out(self): + request, response = self.request('/', headers=( + ('Accept', 'application/json'), + )) + self.assertEqual(response.headers['Content-type'], 'application/json') + + def test_yaml_out(self): + request, response = self.request('/', headers=( + ('Accept', 'application/x-yaml'), + )) + self.assertEqual(response.headers['Content-type'], 'application/x-yaml') + +class TestInFormats(BaseToolsTest): + _cp_config = { + 'tools.hypermedia_in.on': True, + } + + def test_urlencoded_ctype(self): + data = {'valid': 'stuff'} + request, response = self.request('/', method='POST', + body=urllib.urlencode(data), headers=( + ('Content-type', 'application/x-www-form-urlencoded'), + )) + self.assertEqual(response.status, '200 OK') + self.assertDictEqual(request.unserialized_data, data) + + def test_json_ctype(self): + data = {'valid': 'stuff'} + request, response = self.request('/', method='POST', + body=json.dumps(data), headers=( + ('Content-type', 'application/json'), + )) + self.assertEqual(response.status, '200 OK') + self.assertDictEqual(request.unserialized_data, data) + + def test_json_as_text_out(self): + ''' + Some service send JSON as text/plain for compatibility purposes + ''' + data = {'valid': 'stuff'} + request, response = self.request('/', method='POST', + body=json.dumps(data), headers=( + ('Content-type', 'text/plain'), + )) + self.assertEqual(response.status, '200 OK') + self.assertDictEqual(request.unserialized_data, data) + + def test_yaml_ctype(self): + data = {'valid': 'stuff'} + request, response = self.request('/', method='POST', + body=yaml.dump(data), headers=( + ('Content-type', 'application/x-yaml'), + )) + self.assertEqual(response.status, '200 OK') + self.assertDictEqual(request.unserialized_data, data) diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 00000000000..90135757fb9 --- /dev/null +++ b/tests/utils/__init__.py @@ -0,0 +1,93 @@ +# coding: utf-8 +import cherrypy +import mock + +from salt.netapi.rest_cherrypy import app + +from . cptestcase import BaseCherryPyTestCase + +class BaseRestCherryPyTest(BaseCherryPyTestCase): + ''' + A base TestCase subclass for the rest_cherrypy module + + This mocks all interactions with Salt-core and sets up a dummy + (unsubscribed) CherryPy web server. + ''' + __opts__ = None + + @mock.patch('salt.netapi.NetapiClient', autospec=True) + @mock.patch('salt.auth.Resolver', autospec=True) + @mock.patch('salt.auth.LoadAuth', autospec=True) + @mock.patch('salt.utils.event.get_event', autospec=True) + def setUp(self, get_event, LoadAuth, Resolver, NetapiClient): + app.salt.netapi.NetapiClient = NetapiClient + app.salt.auth.Resolver = Resolver + app.salt.auth.LoadAuth = LoadAuth + app.salt.utils.event.get_event = get_event + + # Make local references to mocked objects so individual tests can + # access and modify the mocked interfaces. + self.Resolver = Resolver + self.NetapiClient = NetapiClient + self.get_event = get_event + + __opts__ = self.__opts__ or { + 'external_auth': { + 'auto': { + 'saltdev': [ + '@wheel', + '@runner', + '.*', + ], + } + }, + 'rest_cherrypy': { + 'port': 8000, + 'debug': True, + }, + } + + root, apiopts, conf = app.get_app(__opts__) + + cherrypy.tree.mount(root, '/', conf) + cherrypy.server.unsubscribe() + cherrypy.engine.start() + + def tearDown(self): + cherrypy.engine.exit() + +class Root(object): + ''' + The simplest CherryPy app needed to test individual tools + ''' + exposed = True + + _cp_config = {} + + def GET(self): + return {'return': ['Hello world.']} + + def POST(self, *args, **kwargs): + return {'return': [{'args': args}, {'kwargs': kwargs}]} + +class BaseToolsTest(BaseCherryPyTestCase): + ''' + A base class so tests can selectively turn individual tools on for testing + ''' + conf = { + '/': { + 'request.dispatch': cherrypy.dispatch.MethodDispatcher(), + }, + } + + def setUp(self): + Root._cp_config = self._cp_config + root = Root() + + cherrypy.tree.mount(root, '/', self.conf) + cherrypy.server.unsubscribe() + cherrypy.engine.start() + + def tearDown(self): + cherrypy.engine.exit() + diff --git a/tests/utils/cptestcase.py b/tests/utils/cptestcase.py new file mode 100644 index 00000000000..9ae8d7d6fd2 --- /dev/null +++ b/tests/utils/cptestcase.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2011-2012, Sylvain Hellegouarch +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Sylvain Hellegouarch nor the names of his contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Modified from the original. See the Git history of this file for details. +# https://bitbucket.org/Lawouach/cherrypy-recipes/src/50aff88dc4e24206518ec32e1c32af043f2729da/testing/unit/serverless/cptestcase.py + +from StringIO import StringIO +import unittest +import urllib + +import cherrypy + +# Not strictly speaking mandatory but just makes sense +cherrypy.config.update({'environment': "test_suite"}) + +# This is mandatory so that the HTTP server isn't started +# if you need to actually start (why would you?), simply +# subscribe it back. +cherrypy.server.unsubscribe() + +# simulate fake socket address... they are irrelevant in our context +local = cherrypy.lib.httputil.Host('127.0.0.1', 50000, "") +remote = cherrypy.lib.httputil.Host('127.0.0.1', 50001, "") + +__all__ = ['BaseCherryPyTestCase'] + +class BaseCherryPyTestCase(unittest.TestCase): + def request(self, path='/', method='GET', app_path='', scheme='http', + proto='HTTP/1.1', body=None, qs=None, headers=None, **kwargs): + """ + CherryPy does not have a facility for serverless unit testing. + However this recipe demonstrates a way of doing it by + calling its internal API to simulate an incoming request. + This will exercise the whole stack from there. + + Remember a couple of things: + + * CherryPy is multithreaded. The response you will get + from this method is a thread-data object attached to + the current thread. Unless you use many threads from + within a unit test, you can mostly forget + about the thread data aspect of the response. + + * Responses are dispatched to a mounted application's + page handler, if found. This is the reason why you + must indicate which app you are targetting with + this request by specifying its mount point. + + You can simulate various request settings by setting + the `headers` parameter to a dictionary of headers, + the request's `scheme` or `protocol`. + + .. seealso: http://docs.cherrypy.org/stable/refman/_cprequest.html#cherrypy._cprequest.Response + """ + # This is a required header when running HTTP/1.1 + h = {'Host': '127.0.0.1'} + + # if we had some data passed as the request entity + # let's make sure we have the content-length set + fd = None + if body is not None: + h['content-length'] = '%d' % len(body) + fd = StringIO(body) + + if headers is not None: + h.update(headers) + + # Get our application and run the request against it + app = cherrypy.tree.apps.get(app_path) + if not app: + # XXX: perhaps not the best exception to raise? + raise AssertionError("No application mounted at '%s'" % app_path) + + # Cleanup any previous returned response + # between calls to this method + app.release_serving() + + # Let's fake the local and remote addresses + request, response = app.get_serving(local, remote, scheme, proto) + try: + h = [(k, v) for k, v in h.iteritems()] + response = request.run(method, path, qs, proto, h, fd) + finally: + if fd: + fd.close() + fd = None + + if response.output_status.startswith('500'): + print response.body + raise AssertionError("Unexpected error") + + # collapse the response into a bytestring + response.collapse_body() + return request, response