saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

add niceness control for subprocesses of salt master

Browse source 
under high load these can be useful to tweak for ensuring critical parts
of salt aren't being resource starved.
This commit is contained in:
Matt Phillips 2018-12-18 18:13:48 -05:00 • committed by Daniel Wozniak
parent 821d0def61
commit 3a541e87b0
7 changed files with 261 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

145
doc/ref/configuration/master.rst
View file

@ -1105,6 +1105,151 @@ See the :ref:`2019.2.1 release notes <release-2019-2-1>` for more details.
use_yamlloader_old: False
.. conf_master:: req_server_niceness
``req_server_niceness``
------------------------
.. versionadded:: neon
Default: ``None``
Process priority level of the ReqServer subprocess of the master.
Supported on POSIX platforms only.
.. code-block:: yaml
req_server_niceness: 9
.. conf_master:: pub_server_niceness
``pub_server_niceness``
------------------------
.. versionadded:: neon
Default: ``None``
Process priority level of the PubServer subprocess of the master.
Supported on POSIX platforms only.
.. code-block:: yaml
pub_server_niceness: 9
.. conf_master:: fileserver_update_niceness
``fileserver_update_niceness``
------------------------
.. versionadded:: neon
Default: ``None``
Process priority level of the FileServerUpdate subprocess of the master.
Supported on POSIX platforms only.
.. code-block:: yaml
fileserver_update_niceness: 9
.. conf_master:: maintenance_niceness
``maintenance_niceness``
------------------------
.. versionadded:: neon
Default: ``None``
Process priority level of the Maintenance subprocess of the master.
Supported on POSIX platforms only.
.. code-block:: yaml
maintenance_niceness: 9
.. conf_master:: mworker_niceness
``mworker_niceness``
------------------------
.. versionadded:: neon
Default: ``None``
Process priority level of the MWorker subprocess of the master.
Supported on POSIX platforms only.
.. code-block:: yaml
mworker_niceness: 9
.. conf_master:: mworker_queue_niceness
``mworker_queue_niceness``
------------------------
.. versionadded:: neon
default: ``None``
process priority level of the MWorkerQueue subprocess of the master.
supported on POSIX platforms only.
.. code-block:: yaml
mworker_queue_niceness: 9
.. conf_master:: event_return_niceness
``event_return_niceness``
------------------------
.. versionadded:: neon
default: ``None``
process priority level of the EventReturn subprocess of the master.
supported on POSIX platforms only.
.. code-block:: yaml
event_return_niceness: 9
.. conf_master:: event_publisher_niceness
``event_publisher_niceness``
------------------------
.. versionadded:: neon
default: ``none``
process priority level of the EventPublisher subprocess of the master.
supported on POSIX platforms only.
.. code-block:: yaml
event_publisher_niceness: 9
.. conf_master:: reactor_niceness
``reactor_niceness``
------------------------
.. versionadded:: neon
default: ``None``
process priority level of the Reactor subprocess of the master.
supported on POSIX platforms only.
.. code-block:: yaml
reactor_niceness: 9
.. _salt-ssh-configuration:
Salt-SSH Configuration

20
salt/config/__init__.py
View file

@ -463,6 +463,16 @@ VALID_OPTS = immutabletypes.freeze(
# IPC buffer size
# Refs https://github.com/saltstack/salt/issues/34215
"ipc_write_buffer": int,
# various subprocess niceness levels
"req_server_niceness": (type(None), int),
"pub_server_niceness": (type(None), int),
"fileserver_update_niceness": (type(None), int),
"maintenance_niceness": (type(None), int),
"mworker_niceness": (type(None), int),
"mworker_queue_niceness": (type(None), int),
"event_return_niceness": (type(None), int),
"event_publisher_niceness": (type(None), int),
"reactor_niceness": (type(None), int),
# The number of MWorker processes for a master to startup. This number needs to scale up as
# the number of connected minions increases.
"worker_threads": int,
@ -1413,6 +1423,16 @@ DEFAULT_MASTER_OPTS = immutabletypes.freeze(
"enforce_mine_cache": False,
"ipc_mode": _DFLT_IPC_MODE,
"ipc_write_buffer": _DFLT_IPC_WBUFFER,
# various subprocess niceness levels
"req_server_niceness": None,
"pub_server_niceness": None,
"fileserver_update_niceness": None,
"mworker_niceness": None,
"mworker_queue_niceness": None,
"maintenance_niceness": None,
"event_return_niceness": None,
"event_publisher_niceness": None,
"reactor_niceness": None,
"ipv6": None,
"tcp_master_pub_port": 4512,
"tcp_master_pull_port": 4513,

51
salt/master.py
View file

@ -212,6 +212,12 @@ class Maintenance(salt.utils.process.SignalHandlingProcess):
# Init any values needed by the git ext pillar
self.git_pillar = salt.daemons.masterapi.init_git_pillar(self.opts)
if self.opts["maintenance_niceness"] and not salt.utils.platform.is_windows():
log.info(
"setting Maintenance niceness to %d", self.opts["maintenance_niceness"]
)
os.nice(self.opts["maintenance_niceness"])
self.presence_events = False
if self.opts.get("presence_events", False):
tcp_only = True
@ -511,6 +517,17 @@ class FileserverUpdate(salt.utils.process.SignalHandlingProcess):
Start the update threads
"""
salt.utils.process.appendproctitle(self.__class__.__name__)
if (
self.opts["fileserver_update_niceness"]
and not salt.utils.platform.is_windows()
):
log.info(
"setting FileServerUpdate niceness to %d",
self.opts["fileserver_update_niceness"],
)
os.nice(self.opts["fileserver_update_niceness"])
# Clean out the fileserver backend cache
salt.daemons.masterapi.clean_fsbackend(self.opts)
@ -974,6 +991,13 @@ class ReqServer(salt.utils.process.SignalHandlingProcess):
)
self.opts["worker_threads"] = 1
if self.opts["req_server_niceness"] and not salt.utils.platform.is_windows():
log.info(
"setting ReqServer_ProcessManager niceness to %d",
self.opts["req_server_niceness"],
)
os.nice(self.opts["req_server_niceness"])
# Reset signals to default ones before adding processes to the process
# manager. We don't want the processes being started to inherit those
# signal handlers
@ -1194,6 +1218,33 @@ class MWorker(salt.utils.process.SignalHandlingProcess):
Start a Master Worker
"""
salt.utils.process.appendproctitle(self.name)
# if we inherit req_server level without our own, reset it
if not salt.utils.platform.is_windows():
enforce_mworker_niceness = True
if self.opts["req_server_niceness"]:
if salt.utils.user.get_user() == "root":
log.info(
"%s decrementing inherited ReqServer niceness to 0", self.name
)
log.info(os.nice())
os.nice(-1 * self.opts["req_server_niceness"])
else:
log.error(
"%s unable to decrement niceness for MWorker, not running as root",
self.name,
)
enforce_mworker_niceness = False
# else set what we're explicitly asked for
if enforce_mworker_niceness and self.opts["mworker_niceness"]:
log.info(
"setting %s niceness to %i",
self.name,
self.opts["mworker_niceness"],
)
os.nice(self.opts["mworker_niceness"])
self.clear_funcs = ClearFuncs(self.opts, self.key,)
self.aes_funcs = AESFuncs(self.opts)
salt.utils.crypt.reinit_crypto()

7
salt/transport/tcp.py
View file

@ -717,6 +717,13 @@ class TCPReqServerChannel(
payload_handler: function to call with your payloads
"""
if self.opts["pub_server_niceness"] and not salt.utils.platform.is_windows():
log.info(
"setting Publish daemon niceness to %i",
self.opts["pub_server_niceness"],
)
os.nice(self.opts["pub_server_niceness"])
self.payload_handler = payload_handler
self.io_loop = io_loop
self.serial = salt.payload.Serial(self.opts)

15
salt/transport/zeromq.py
View file

@ -684,6 +684,13 @@ class ZeroMQReqServerChannel(
self._start_zmq_monitor()
self.workers = self.context.socket(zmq.DEALER)
if self.opts["mworker_queue_niceness"] and not salt.utils.platform.is_windows():
log.info(
"setting mworker_queue niceness to %d",
self.opts["mworker_queue_niceness"],
)
os.nice(self.opts["mworker_queue_niceness"])
if self.opts.get("ipc_mode", "") == "tcp":
self.w_uri = "tcp://127.0.0.1:{0}".format(
self.opts.get("tcp_master_workers", 4515)
@ -941,6 +948,14 @@ class ZeroMQPubServerChannel(salt.transport.server.PubServerChannel):
Bind to the interface specified in the configuration file
"""
salt.utils.process.appendproctitle(self.__class__.__name__)
if self.opts["pub_server_niceness"] and not salt.utils.platform.is_windows():
log.info(
"setting Publish daemon niceness to %i",
self.opts["pub_server_niceness"],
)
os.nice(self.opts["pub_server_niceness"])
if log_queue:
salt.log.setup.set_multiprocessing_logging_queue(log_queue)
salt.log.setup.setup_multiprocessing_logging(log_queue)

18
salt/utils/event.py
View file

@ -1116,6 +1116,17 @@ class EventPublisher(salt.utils.process.SignalHandlingProcess):
Bind the pub and pull sockets for events
"""
salt.utils.process.appendproctitle(self.__class__.__name__)
if (
self.opts["event_publisher_niceness"]
and not salt.utils.platform.is_windows()
):
log.info(
"setting EventPublisher niceness to %i",
self.opts["event_publisher_niceness"],
)
os.nice(self.opts["event_publisher_niceness"])
self.io_loop = salt.ext.tornado.ioloop.IOLoop()
with salt.utils.asynchronous.current_ioloop(self.io_loop):
if self.opts["ipc_mode"] == "tcp":
@ -1282,6 +1293,13 @@ class EventReturn(salt.utils.process.SignalHandlingProcess):
Spin up the multiprocess event returner
"""
salt.utils.process.appendproctitle(self.__class__.__name__)
if self.opts["event_return_niceness"] and not salt.utils.platform.is_windows():
log.info(
"setting EventReturn niceness to %i", self.opts["event_return_niceness"]
)
os.nice(self.opts["event_return_niceness"])
self.event = get_event("master", opts=self.opts, listen=True)
events = self.event.iter_events(full=True)
self.event.fire_event({}, "salt/event_listen/start")

5
salt/utils/reactor.py
View file

@ -10,6 +10,7 @@ from __future__ import absolute_import, print_function, unicode_literals
import fnmatch
import glob
import logging
import os
# Import salt libs
import salt.client
@ -236,6 +237,10 @@ class Reactor(salt.utils.process.SignalHandlingProcess, salt.state.Compiler):
"""
salt.utils.process.appendproctitle(self.__class__.__name__)
if self.opts["reactor_niceness"] and not salt.utils.platform.is_windows():
log.info("Reactor setting niceness to %i", self.opts["reactor_niceness"])
os.nice(self.opts["reactor_niceness"])
# instantiate some classes inside our new process
with salt.utils.event.get_event(
self.opts["__role"],