saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #6573 from erchn/develop

Browse source 
Performance increase for large file.recurse directories
This commit is contained in:
Thomas S Hatch 2013-08-08 12:17:51 -07:00
commit 3e1df4f901
5 changed files with 39 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
doc/man/salt.7
View file

@ -11941,7 +11941,7 @@ salt \(aq*\(aq cp.is_cached salt://path/to/file
.UNINDENT
.INDENT 0.0
.TP
.B salt.modules.cp.list_master(env=\(aqbase\(aq)
.B salt.modules.cp.list_master(env=\(aqbase\(aq, prefix=\(aq\(aq)
List all of the files stored on the master
.sp
CLI Example:
@ -11954,7 +11954,7 @@ salt \(aq*\(aq cp.list_master
.UNINDENT
.INDENT 0.0
.TP
.B salt.modules.cp.list_master_dirs(env=\(aqbase\(aq)
.B salt.modules.cp.list_master_dirs(env=\(aqbase\(aq, prefix=\(aq\(aq)
List all of the directories stored on the master
.sp
CLI Example:

36
salt/fileclient.py
View file

@ -102,7 +102,7 @@ class Client(object):
'''
raise NotImplementedError
def file_list_emptydirs(self, env='base'):
def file_list_emptydirs(self, env='base', prefix=''):
'''
List the empty dirs
'''
@ -207,13 +207,13 @@ class Client(object):
ldest = self._file_local_list(localfilesdest)
return sorted(fdest.union(ldest))
def file_list(self, env='base'):
def file_list(self, env='base', prefix=''):
'''
This function must be overwritten
'''
return []
def dir_list(self, env='base'):
def dir_list(self, env='base', prefix=''):
'''
This function must be overwritten
'''
@ -461,47 +461,56 @@ class LocalClient(Client):
return ''
return fnd['path']
def file_list(self, env='base'):
def file_list(self, env='base', prefix=''):
'''
Return a list of files in the given environment
with optional relative prefix path to limit directory traversal
'''
ret = []
if env not in self.opts['file_roots']:
return ret
for path in self.opts['file_roots'][env]:
if prefix:
path = os.path.join(path, prefix)
for root, dirs, files in os.walk(path, followlinks=True):
for fname in files:
ret.append(
os.path.relpath(
os.path.join(root, fname),
os.path.join(root, prefix, fname),
path
)
)
return ret
def file_list_emptydirs(self, env='base'):
def file_list_emptydirs(self, env='base', prefix=''):
'''
List the empty dirs in the file_roots
with optional relative prefix path to limit directory traversal
'''
ret = []
if env not in self.opts['file_roots']:
return ret
for path in self.opts['file_roots'][env]:
if prefix:
path = os.path.join(path, prefix)
for root, dirs, files in os.walk(path, followlinks=True):
if len(dirs) == 0 and len(files) == 0:
ret.append(os.path.relpath(root, path))
ret.append(os.path.relpath(root, os.path.join(prefix, path)))
return ret
def dir_list(self, env='base'):
def dir_list(self, env='base', prefix=''):
'''
List the dirs in the file_roots
with optional relative prefix path to limit directory traversal
'''
ret = []
if env not in self.opts['file_roots']:
return ret
for path in self.opts['file_roots'][env]:
if prefix:
path = os.path.join(path, prefix)
for root, dirs, files in os.walk(path, followlinks=True):
ret.append(os.path.relpath(root, path))
ret.append(os.path.relpath(root, os.path.join(prefix, path)))
return ret
def hash_file(self, path, env='base'):
@ -667,11 +676,12 @@ class RemoteClient(Client):
fn_.close()
return dest
def file_list(self, env='base'):
def file_list(self, env='base', prefix=''):
'''
List the files on the master
'''
load = {'env': env,
'prefix': prefix,
'cmd': '_file_list'}
try:
return self.auth.crypticle.loads(
@ -683,11 +693,12 @@ class RemoteClient(Client):
except SaltReqTimeoutError:
return ''
def file_list_emptydirs(self, env='base'):
def file_list_emptydirs(self, env='base', prefix=''):
'''
List the empty dirs on the master
'''
load = {'env': env,
'prefix': prefix,
'cmd': '_file_list_emptydirs'}
try:
return self.auth.crypticle.loads(
@ -699,11 +710,12 @@ class RemoteClient(Client):
except SaltReqTimeoutError:
return ''
def dir_list(self, env='base'):
def dir_list(self, env='base', prefix=''):
'''
List the dirs on the master
'''
load = {'env': env,
'prefix': prefix,
'cmd': '_dir_list'}
try:
return self.auth.crypticle.loads(

9
salt/fileserver/roots.py
View file

@ -99,10 +99,11 @@ def file_list(load):
return ret
for path in __opts__['file_roots'][load['env']]:
path = os.path.join(path, load['prefix'])
for root, dirs, files in os.walk(path, followlinks=True):
for fname in files:
rel_fn = os.path.relpath(
os.path.join(root, fname),
os.path.join(root, load['prefix'], fname),
path
)
if not salt.fileserver.is_file_ignored(__opts__, rel_fn):
@ -118,11 +119,12 @@ def file_list_emptydirs(load):
if load['env'] not in __opts__['file_roots']:
return ret
for path in __opts__['file_roots'][load['env']]:
path = os.path.join(path, load['prefix'])
for root, dirs, files in os.walk(path, followlinks=True):
if len(dirs) == 0 and len(files) == 0:
rel_fn = os.path.relpath(root, path)
if not salt.fileserver.is_file_ignored(__opts__, rel_fn):
ret.append(rel_fn)
ret.append(os.path.join(load['prefix'], rel_fn))
return ret
@ -134,6 +136,7 @@ def dir_list(load):
if load['env'] not in __opts__['file_roots']:
return ret
for path in __opts__['file_roots'][load['env']]:
path = os.path.join(path, load['prefix'])
for root, dirs, files in os.walk(path, followlinks=True):
ret.append(os.path.relpath(root, path))
ret.append(os.path.relpath(root, os.path.join(load['prefix'], path)))
return ret

8
salt/modules/cp.py
View file

@ -312,7 +312,7 @@ def list_states(env='base'):
return __context__['cp.fileclient'].list_states(env)
def list_master(env='base'):
def list_master(env='base', prefix=''):
'''
List all of the files stored on the master
@ -321,10 +321,10 @@ def list_master(env='base'):
salt '*' cp.list_master
'''
_mk_client()
return __context__['cp.fileclient'].file_list(env)
return __context__['cp.fileclient'].file_list(env, prefix)
def list_master_dirs(env='base'):
def list_master_dirs(env='base', prefix=''):
'''
List all of the directories stored on the master
@ -333,7 +333,7 @@ def list_master_dirs(env='base'):
salt '*' cp.list_master_dirs
'''
_mk_client()
return __context__['cp.fileclient'].dir_list(env)
return __context__['cp.fileclient'].dir_list(env, prefix)
def list_minion(env='base'):

10
salt/states/file.py
View file

@ -1241,7 +1241,7 @@ def recurse(name,
if not _src_path:
pass
elif _src_path.strip('/') not in __salt__['cp.list_master_dirs'](env):
elif _src_path.strip('/') not in __salt__['cp.list_master_dirs'](env, srcpath):
ret['result'] = False
ret['comment'] = (
'The source: {0} does not exist on the master'.format(source)
@ -1353,11 +1353,9 @@ def recurse(name,
#we're searching for things that start with this *directory*.
# use '/' since #master only runs on POSIX
srcpath = srcpath + '/'
for fn_ in __salt__['cp.list_master'](env):
for fn_ in __salt__['cp.list_master'](env, srcpath):
if not fn_.strip():
continue
if not fn_.startswith(srcpath):
continue
# fn_ here is the absolute (from file_roots) source path of
# the file to copy from; it is either a normal file or an
@ -1393,10 +1391,8 @@ def recurse(name,
manage_file(dest, src)
if include_empty:
mdirs = __salt__['cp.list_master_dirs'](env)
mdirs = __salt__['cp.list_master_dirs'](env, srcpath)
for mdir in mdirs:
if not mdir.startswith(srcpath):
continue
if not _check_include_exclude(os.path.relpath(mdir, srcpath),
include_pat,
exclude_pat):