Add runner to sync eauth_token modules, fix masterapi daemon to use token module

doc/ref/configuration/master.rst

@@ -234,6 +234,7 @@ Valid options:
   - clouds
   - tops
   - roster
+  - tokens
 
 .. conf_master:: module_dirs
 

salt/auth/__init__.py

@@ -236,10 +236,22 @@ class LoadAuth(object):
         if tdata.get('expire', '0') < time.time():
             rm_tok = True
         if rm_tok:
-            self.tokens["{0}.rm_token".format(self.opts['eauth_tokens'])](self.opts, tok)
+            self.rm_token(tok)
 
         return tdata
 
+    def list_tokens(self):
+        '''
+        List all tokens in eauth_tokn storage.
+        '''
+        return self.tokens["{0}.list_tokens".format(self.opts['eauth_tokens'])](self.opts)
+
+    def rm_token(self, tok):
+        '''
+        Remove the given token from token storage.
+        '''
+        self.tokens["{0}.rm_token".format(self.opts['eauth_tokens'])](self.opts, tok)
+
     def authenticate_token(self, load):
         '''
         Authenticate a user by the token specified in load.

salt/daemons/masterapi.py

@@ -152,22 +152,11 @@ def clean_expired_tokens(opts):
     '''
     Clean expired tokens from the master
     '''
-    serializer = salt.payload.Serial(opts)
-    for (dirpath, dirnames, filenames) in os.walk(opts['token_dir']):
-        for token in filenames:
-            token_path = os.path.join(dirpath, token)
-            with salt.utils.files.fopen(token_path, 'rb') as token_file:
-                try:
-                    token_data = serializer.loads(token_file.read())
-                except msgpack.UnpackValueError:
-                    # Bad token file or empty. Remove.
-                    os.remove(token_path)
-                    return
-                if 'expire' not in token_data or token_data.get('expire', 0) < time.time():
-                    try:
-                        os.remove(token_path)
-                    except (IOError, OSError):
-                        pass
+    loadauth = salt.auth.LoadAuth(opts)
+    for tok in loadauth.list_tokens():
+        token_data = loadauth.get_tok(tok)
+        if 'expire' not in token_data or token_Data.get('expire', 0) < time.time():
+            loadauth.rm_token(tok)
 
 
 def clean_pub_auth(opts):

salt/runners/saltutil.py

@@ -59,6 +59,7 @@ def sync_all(saltenv='base', extmod_whitelist=None, extmod_blacklist=None):
     ret['cache'] = sync_cache(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist)
     ret['fileserver'] = sync_fileserver(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist)
     ret['tops'] = sync_tops(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist)
+    ret['tokens'] = sync_eauth_tokens(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist)
     return ret
 
 
@@ -524,3 +525,29 @@ def sync_roster(saltenv='base', extmod_whitelist=None, extmod_blacklist=None):
     '''
     return salt.utils.extmods.sync(__opts__, 'roster', saltenv=saltenv, extmod_whitelist=extmod_whitelist,
                                    extmod_blacklist=extmod_blacklist)[0]
+
+
+def sync_eauth_tokens(saltenv='base', extmod_whitelist=None, extmod_blacklist=None):
+    '''
+    .. versionadded:: 2017.7.2
+
+    Sync eauth token modules from ``salt://_tokens`` to the master
+
+    saltenv : base
+        The fileserver environment from which to sync. To sync from more than
+        one environment, pass a comma-separated list.
+
+    extmod_whitelist : None
+        comma-seperated list of modules to sync
+
+    extmod_blacklist : None
+        comma-seperated list of modules to blacklist based on type
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt-run saltutil.sync_eauth_tokens
+    '''
+    return salt.utils.extmods.sync(__opts__, 'tokens', saltenv=saltenv, extmod_whitelist=extmod_whitelist,
+                                   extmod_blacklist=extmod_blacklist)[0]

salt/tokens/__init__.py

@@ -12,4 +12,6 @@
 
     :rm_token: remove the given token from storage
 
+    :list_tokens: list all tokens in storage
+
 '''

salt/tokens/localfs.py

@@ -81,3 +81,17 @@ def rm_token(opts, tok):
         return {}
     except (IOError, OSError):
         log.warning('Could not remove token {0}'.format(tok))
+
+
+def list_tokens(opts):
+    '''
+    List all tokens in the store.
+
+    :param opts: Salt master config options
+    :returns: List of dicts (tokens)
+    '''
+    ret = []
+    for (dirpath, dirnames, filenames) in os.walk(opts['token_dir']):
+        for token in filenames:
+            ret.append(token)
+    return ret

salt/tokens/rediscluster.py

@@ -124,3 +124,22 @@ def rm_token(opts, tok):
         return {}
     except Exception as err:
         log.warning("Could not remove token {0}: {1}".format(tok, err))
+
+
+def list_tokens(opts):
+    '''
+    List all tokens in the store.
+
+    :param opts: Salt master config options
+    :returns: List of dicts (token_data)
+    '''
+    ret = []
+    redis_client = _redis_client(opts)
+    if not redis_client:
+        return []
+    serial = salt.payload.Serial(opts)
+    try:
+        return [k.decode('utf8') for k in redis_client.keys()]
+    except Exception as err:
+        log.warning("Failed to list keys: {0}".format(err))
+        return []