saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Make Pillar no longer munge file_roots

Browse source 
Pillar and the local fileclient used to use the same class, but this
changed a few years ago with the introduction of FSClient (which enabled
non-roots fileserver backends in masterless Salt). Since Pillar is the
only thing that uses this client, there is no longer a reason to make
Pillar modify file_roots.
This commit is contained in:
Erik Johnson 2018-10-10 13:10:50 -05:00
parent c912d3a288
commit 58f5fde03c
No known key found for this signature in database
GPG key ID: 5E5583C437808F3F
5 changed files with 52 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
doc/ref/configuration/master.rst
View file

@ -2436,6 +2436,12 @@ Master will not be returned to the Minion.
------------------------------
.. versionadded:: 2014.1.0
.. deprecated:: 2018.3.4
This option is now ignored. Firstly, it only traversed
:conf_master:`file_roots`, which means it did not work for the other
fileserver backends. Secondly, since this option was added we have added
caching to the code that traverses the file_roots (and gitfs, etc.), which
greatly reduces the amount of traversal that is done.
Default: ``False``

103
salt/fileclient.py
View file

@ -61,7 +61,7 @@ def get_file_client(opts, pillar=False):
return {
'remote': RemoteClient,
'local': FSClient,
'pillar': LocalClient,
'pillar': PillarClient,
}.get(client, RemoteClient)(opts)
@ -347,58 +347,17 @@ class Client(object):
Return a list of all available sls modules on the master for a given
environment
'''
limit_traversal = self.opts.get('fileserver_limit_traversal', False)
states = []
if limit_traversal:
if saltenv not in self.opts['file_roots']:
log.warning(
'During an attempt to list states for saltenv \'%s\', '
'the environment could not be found in the configured '
'file roots', saltenv
)
return states
for path in self.opts['file_roots'][saltenv]:
for root, dirs, files in os.walk(path, topdown=True): # future lint: disable=blacklisted-function
root = salt.utils.data.decode(root)
files = salt.utils.data.decode(files)
log.debug(
'Searching for states in dirs %s and files %s',
salt.utils.data.decode(dirs), files
)
if not [filename.endswith('.sls') for filename in files]:
# Use shallow copy so we don't disturb the memory used
# by os.walk. Otherwise this breaks!
del dirs[:]
else:
for found_file in files:
stripped_root = os.path.relpath(root, path)
if salt.utils.platform.is_windows():
stripped_root = stripped_root.replace('\\', '/')
stripped_root = stripped_root.replace('/', '.')
if found_file.endswith(('.sls')):
if found_file.endswith('init.sls'):
if stripped_root.endswith('.'):
stripped_root = stripped_root.rstrip('.')
states.append(stripped_root)
else:
if not stripped_root.endswith('.'):
stripped_root += '.'
if stripped_root.startswith('.'):
stripped_root = stripped_root.lstrip('.')
states.append(stripped_root + found_file[:-4])
else:
for path in self.file_list(saltenv):
if salt.utils.platform.is_windows():
path = path.replace('\\', '/')
if path.endswith('.sls'):
# is an sls module!
if path.endswith('/init.sls'):
states.append(path.replace('/', '.')[:-9])
else:
states.append(path.replace('/', '.')[:-4])
return states
states = set()
for path in self.file_list(saltenv):
if salt.utils.platform.is_windows():
path = path.replace('\\', '/')
if path.endswith('.sls'):
# is an sls module!
if path.endswith('/init.sls'):
states.add(path.replace('/', '.')[:-9])
else:
states.add(path.replace('/', '.')[:-4])
return sorted(states)
def get_state(self, sls, saltenv, cachedir=None):
'''
@ -844,13 +803,10 @@ class Client(object):
)
class LocalClient(Client):
class PillarClient(Client):
'''
Use the local_roots option to parse a local file root
Used by pillar to handle fileclient requests
'''
def __init__(self, opts):
Client.__init__(self, opts)
def _find_file(self, path, saltenv='base'):
'''
Locate the file path
@ -858,12 +814,12 @@ class LocalClient(Client):
fnd = {'path': '',
'rel': ''}
if saltenv not in self.opts['file_roots']:
if saltenv not in self.opts['pillar_roots']:
return fnd
if salt.utils.url.is_escaped(path):
# The path arguments are escaped
path = salt.utils.url.unescape(path)
for root in self.opts['file_roots'][saltenv]:
for root in self.opts['pillar_roots'][saltenv]:
full = os.path.join(root, path)
if os.path.isfile(full):
fnd['path'] = full
@ -896,10 +852,10 @@ class LocalClient(Client):
with optional relative prefix path to limit directory traversal
'''
ret = []
if saltenv not in self.opts['file_roots']:
if saltenv not in self.opts['pillar_roots']:
return ret
prefix = prefix.strip('/')
for path in self.opts['file_roots'][saltenv]:
for path in self.opts['pillar_roots'][saltenv]:
for root, dirs, files in salt.utils.path.os_walk(
os.path.join(path, prefix), followlinks=True
):
@ -912,14 +868,14 @@ class LocalClient(Client):
def file_list_emptydirs(self, saltenv='base', prefix=''):
'''
List the empty dirs in the file_roots
List the empty dirs in the pillar_roots
with optional relative prefix path to limit directory traversal
'''
ret = []
prefix = prefix.strip('/')
if saltenv not in self.opts['file_roots']:
if saltenv not in self.opts['pillar_roots']:
return ret
for path in self.opts['file_roots'][saltenv]:
for path in self.opts['pillar_roots'][saltenv]:
for root, dirs, files in salt.utils.path.os_walk(
os.path.join(path, prefix), followlinks=True
):
@ -931,14 +887,14 @@ class LocalClient(Client):
def dir_list(self, saltenv='base', prefix=''):
'''
List the dirs in the file_roots
List the dirs in the pillar_roots
with optional relative prefix path to limit directory traversal
'''
ret = []
if saltenv not in self.opts['file_roots']:
if saltenv not in self.opts['pillar_roots']:
return ret
prefix = prefix.strip('/')
for path in self.opts['file_roots'][saltenv]:
for path in self.opts['pillar_roots'][saltenv]:
for root, dirs, files in salt.utils.path.os_walk(
os.path.join(path, prefix), followlinks=True
):
@ -965,7 +921,7 @@ class LocalClient(Client):
def hash_file(self, path, saltenv='base'):
'''
Return the hash of a file, to get the hash of a file in the file_roots
Return the hash of a file, to get the hash of a file in the pillar_roots
prepend the path with salt://<file on server> otherwise, prepend the
file with / for a local file.
'''
@ -988,7 +944,7 @@ class LocalClient(Client):
def hash_and_stat_file(self, path, saltenv='base'):
'''
Return the hash of a file, to get the hash of a file in the file_roots
Return the hash of a file, to get the hash of a file in the pillar_roots
prepend the path with salt://<file on server> otherwise, prepend the
file with / for a local file.
@ -1034,7 +990,7 @@ class LocalClient(Client):
Return the available environments
'''
ret = []
for saltenv in self.opts['file_roots']:
for saltenv in self.opts['pillar_roots']:
ret.append(saltenv)
return ret
@ -1428,6 +1384,11 @@ class FSClient(RemoteClient):
self.auth = DumbAuth()
# Provide backward compatibility for anyone directly using LocalClient (but no
# one should be doing this).
LocalClient = FSClient
class DumbAuth(object):
'''
The dumbauth class is used to stub out auth calls fired from the FSClient

28
salt/pillar/__init__.py
View file

@ -345,8 +345,6 @@ class Pillar(object):
if pillarenv is None:
if opts.get('pillarenv_from_saltenv', False):
opts['pillarenv'] = saltenv
# Store the file_roots path so we can restore later. Issue 5449
self.actual_file_roots = opts['file_roots']
# use the local file client
self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, pillarenv=pillarenv)
self.saltenv = saltenv
@ -369,9 +367,6 @@ class Pillar(object):
self.matcher = salt.minion.Matcher(self.opts, self.functions)
self.rend = salt.loader.render(self.opts, self.functions)
ext_pillar_opts = copy.deepcopy(self.opts)
# Fix self.opts['file_roots'] so that ext_pillars know the real
# location of file_roots. Issue 5951
ext_pillar_opts['file_roots'] = self.actual_file_roots
# Keep the incoming opts ID intact, ie, the master id
if 'id' in opts:
ext_pillar_opts['id'] = opts['id']
@ -438,7 +433,6 @@ class Pillar(object):
The options need to be altered to conform to the file client
'''
opts = copy.deepcopy(opts_in)
opts['file_roots'] = opts['pillar_roots']
opts['file_client'] = 'local'
if not grains:
opts['grains'] = {}
@ -463,15 +457,15 @@ class Pillar(object):
opts['ext_pillar'].append(self.ext)
else:
opts['ext_pillar'] = [self.ext]
if '__env__' in opts['file_roots']:
if '__env__' in opts['pillar_roots']:
env = opts.get('pillarenv') or opts.get('saltenv') or 'base'
if env not in opts['file_roots']:
if env not in opts['pillar_roots']:
log.debug("pillar environment '%s' maps to __env__ pillar_roots directory", env)
opts['file_roots'][env] = opts['file_roots'].pop('__env__')
opts['pillar_roots'][env] = opts['pillar_roots'].pop('__env__')
else:
log.debug("pillar_roots __env__ ignored (environment '%s' found in pillar_roots)",
env)
opts['file_roots'].pop('__env__')
opts['pillar_roots'].pop('__env__')
return opts
def _get_envs(self):
@ -479,8 +473,8 @@ class Pillar(object):
Pull the file server environments out of the master options
'''
envs = set(['base'])
if 'file_roots' in self.opts:
envs.update(list(self.opts['file_roots']))
if 'pillar_roots' in self.opts:
envs.update(list(self.opts['pillar_roots']))
return envs
def get_tops(self):
@ -496,11 +490,11 @@ class Pillar(object):
if self.opts['pillarenv']:
# If the specified pillarenv is not present in the available
# pillar environments, do not cache the pillar top file.
if self.opts['pillarenv'] not in self.opts['file_roots']:
if self.opts['pillarenv'] not in self.opts['pillar_roots']:
log.debug(
'pillarenv \'%s\' not found in the configured pillar '
'environments (%s)',
self.opts['pillarenv'], ', '.join(self.opts['file_roots'])
self.opts['pillarenv'], ', '.join(self.opts['pillar_roots'])
)
else:
top = self.client.cache_file(self.opts['state_top'], self.opts['pillarenv'])
@ -1010,8 +1004,6 @@ class Pillar(object):
mopts = dict(self.opts)
if 'grains' in mopts:
mopts.pop('grains')
# Restore the actual file_roots path. Issue 5449
mopts['file_roots'] = self.actual_file_roots
mopts['saltversion'] = __version__
pillar['master'] = mopts
if 'pillar' in self.opts and self.opts.get('ssh_merge_pillar', False):
@ -1038,10 +1030,6 @@ class Pillar(object):
if decrypt_errors:
pillar.setdefault('_errors', []).extend(decrypt_errors)
# Reset the file_roots for the renderers
for mod_name in sys.modules:
if mod_name.startswith('salt.loaded.int.render.'):
sys.modules[mod_name].__opts__['file_roots'] = self.actual_file_roots
return pillar
def decrypt_pillar(self, pillar):

8
salt/utils/jinja.py
View file

@ -59,16 +59,16 @@ class SaltCacheLoader(BaseLoader):
self.opts = opts
self.saltenv = saltenv
self.encoding = encoding
if self.opts['file_roots'] is self.opts['pillar_roots']:
if saltenv not in self.opts['file_roots']:
self.pillar_rend = pillar_rend
if self.pillar_rend:
if saltenv not in self.opts['pillar_roots']:
self.searchpath = []
else:
self.searchpath = opts['file_roots'][saltenv]
self.searchpath = opts['pillar_roots'][saltenv]
else:
self.searchpath = [os.path.join(opts['cachedir'], 'files', saltenv)]
log.debug('Jinja search path: %s', self.searchpath)
self.cached = []
self.pillar_rend = pillar_rend
self._file_client = None
# Instantiate the fileclient
self.file_client()

4
tests/integration/pillar/test_git_pillar.py
View file

@ -101,13 +101,13 @@ from salt.utils.gitfs import (
# Check for requisite components
try:
HAS_GITPYTHON = GITPYTHON_VERSION >= GITPYTHON_MINVER
except ImportError:
except Exception:
HAS_GITPYTHON = False
try:
HAS_PYGIT2 = PYGIT2_VERSION >= PYGIT2_MINVER \
and LIBGIT2_VERSION >= LIBGIT2_MINVER
except AttributeError:
except Exception:
HAS_PYGIT2 = False
HAS_SSHD = bool(salt.utils.path.which('sshd'))