Restart maintenance process periodically.

Unfortunatly pygit2 still has enough memory leaks that we can't use it
in long running processes. Restart the maintenance process peridically
to clean up the memory consumed by pygit, and others.

Add fileserver_interval and maintenance_interval to configure how often
these processes restart.

salt/config/__init__.py

@@ -987,6 +987,10 @@ VALID_OPTS = immutabletypes.freeze(
         "pass_gnupghome": str,
         # pass renderer: Set PASSWORD_STORE_DIR env for Pass
         "pass_dir": str,
+        # Maintenence process restart interval
+        "maintenance_interval": int,
+        # Fileserver process restart interval
+        "fileserver_interval": int,
     }
 )
 
@@ -1635,6 +1639,8 @@ DEFAULT_MASTER_OPTS = immutabletypes.freeze(
         "pass_gnupghome": "",
         "pass_dir": "",
         "netapi_enable_clients": [],
+        "maintenence_interval": 3600,
+        "fileserver_interval": 3600,
     }
 )
 

salt/master.py

@@ -186,6 +186,7 @@ class Maintenance(salt.utils.process.SignalHandlingProcess):
         # Track key rotation intervals
         self.rotate = int(time.time())
         # A serializer for general maint operations
+        self.restart_interval = int(self.opts["maintenance_interval"])
 
     def _post_fork_init(self):
         """
@@ -243,21 +244,28 @@ class Maintenance(salt.utils.process.SignalHandlingProcess):
         # init things that need to be done after the process is forked
         self._post_fork_init()
 
-        # Make Start Times
-        last = int(time.time())
+        # Start of process for maintenance process restart interval
+        start = time.time()
+
+        # Unset last value will cause the interval items to run on the first
+        # loop iteration. This ensurs we always run them even if
+        # maintenance_interval happens to be less than loop_interval or
+        # git_update_interval
+        last = None
+
         # update git_pillar on first loop
         last_git_pillar_update = 0
+        now = int(time.time())
 
         git_pillar_update_interval = self.opts.get("git_pillar_update_interval", 0)
         old_present = set()
-        while True:
-            now = int(time.time())
+        while time.time() - start < self.restart_interval:
             log.trace("Running maintenance routines")
-            if (now - last) >= self.loop_interval:
+            if not last or (now - last) >= self.loop_interval:
                 salt.daemons.masterapi.clean_old_jobs(self.opts)
                 salt.daemons.masterapi.clean_expired_tokens(self.opts)
                 salt.daemons.masterapi.clean_pub_auth(self.opts)
-            if (now - last_git_pillar_update) >= git_pillar_update_interval:
+            if not last or (now - last_git_pillar_update) >= git_pillar_update_interval:
                 last_git_pillar_update = now
                 self.handle_git_pillar()
             self.handle_schedule()
@@ -266,6 +274,7 @@ class Maintenance(salt.utils.process.SignalHandlingProcess):
             self.handle_key_rotate(now)
             salt.utils.verify.check_max_open_files(self.opts)
             last = now
+            now = int(time.time())
             time.sleep(self.loop_interval)
 
     def handle_key_cache(self):
@@ -462,7 +471,7 @@ class FileserverUpdate(salt.utils.process.SignalHandlingProcess):
                 )
 
     @classmethod
-    def update(cls, interval, backends, timeout=300):
+    def update(cls, interval, backends, timeout):
         """
         Threading target which handles all updates for a given wait interval
         """
@@ -503,7 +512,11 @@ class FileserverUpdate(salt.utils.process.SignalHandlingProcess):
         for interval in self.buckets:
             self.update_threads[interval] = threading.Thread(
                 target=self.update,
-                args=(interval, self.buckets[interval]),
+                args=(
+                    interval,
+                    self.buckets[interval],
+                    self.opts["fileserver_interval"],
+                ),
             )
             self.update_threads[interval].start()
 

tests/pytests/unit/test_master.py

@@ -21,7 +21,39 @@ def encrypted_requests(tmp_path):
     )
 
 
+def test_maintenance_duration():
+    """
+    Validate Maintenance process duration.
+    """
+    opts = {
+        "loop_interval": 10,
+        "maintenence_interval": 1,
+        "cachedir": "/tmp",
+        "sock_dir": "/tmp",
+        "maintenance_niceness": 1,
+        "key_cache": "sched",
+        "conf_file": "",
+        "master_job_cache": "",
+        "pki_dir": "/tmp",
+        "eauth_tokens": "",
+    }
+    mp = salt.master.Maintenance(opts)
+    with patch("salt.utils.verify.check_max_open_files") as check_files, patch.object(
+        mp, "handle_key_cache"
+    ) as handle_key_cache, patch("salt.daemons") as salt_daemons, patch.object(
+        mp, "handle_git_pillar"
+    ) as handle_git_pillar:
+        mp.run()
+    assert salt_daemons.masterapi.clean_old_jobs.called
+    assert salt_daemons.masterapi.clean_expired_tokens.called
+    assert salt_daemons.masterapi.clean_pub_auth.called
+    assert handle_git_pillar.called
+
+
 def test_fileserver_duration():
+    """
+    Validate Fileserver process duration.
+    """
     with patch("salt.master.FileserverUpdate._do_update") as update:
         start = time.time()
         salt.master.FileserverUpdate.update(1, {}, 1)