saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge branch '2017.7' into loader-docs

Browse source
This commit is contained in:
Jamie Bliss 2018-11-30 19:20:11 -05:00 • committed by GitHub
commit 9d596298ea
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 178 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

122
.ci/lint
View file

@ -3,7 +3,7 @@ pipeline {
options {
timestamps()
ansiColor('xterm')
timeout(time: 1, unit: 'HOURS')
timeout(time: 3, unit: 'HOURS')
}
environment {
PYENV_ROOT = "/usr/local/pyenv"
@ -14,7 +14,7 @@ pipeline {
stage('github-pending') {
steps {
githubNotify credentialsId: 'test-jenkins-credentials',
description: 'Testing lint...',
description: 'Python lint on changes begins...',
status: 'PENDING',
context: "jenkins/pr/lint"
}
@ -24,12 +24,15 @@ pipeline {
sh '''
# Need -M to detect renames otherwise they are reported as Delete and Add, need -C to detect copies, -C includes -M
# -M is on by default in git 2.9+
git diff --name-status -l99999 -C "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME" > file-list-status.log
git diff --name-status -l99999 -C "origin/$CHANGE_TARGET" > file-list-status.log
# the -l increase the search limit, lets use awk so we do not need to repeat the search above.
gawk 'BEGIN {FS="\\t"} {if ($1 != "D") {print $NF}}' file-list-status.log > file-list-changed.log
gawk 'BEGIN {FS="\\t"} {if ($1 == "D") {print $NF}}' file-list-status.log > file-list-deleted.log
(git diff --name-status -l99999 -C "origin/$CHANGE_TARGET";echo "---";git diff --name-status -l99999 -C "origin/$BRANCH_NAME";printenv|grep -E '=[0-9a-z]{40,}+$|COMMIT=|BRANCH') > file-list-experiment.log
touch pylint-report-salt.log pylint-report-tests.log
(git diff --name-status -l99999 -C "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME";echo "---";git diff --name-status -l99999 -C "origin/$BRANCH_NAME";printenv|grep -E '=[0-9a-z]{40,}+$|COMMIT=|BRANCH') > file-list-experiment.log
echo 254 > pylint-salt-chg.exit # assume failure
echo 254 > pylint-salt-full.exit # assume failure
echo 254 > pylint-tests-chg.exit # assume failure
echo 254 > pylint-tests-full.exit # assume failure
eval "$(pyenv init -)"
pyenv --version
pyenv install --skip-existing 2.7.14
@ -41,63 +44,128 @@ pipeline {
archiveArtifacts artifacts: 'file-list-status.log,file-list-changed.log,file-list-deleted.log,file-list-experiment.log'
}
}
stage('linting') {
failFast false
stage('linting chg') {
parallel {
stage('salt linting') {
stage('lint salt chg') {
when {
expression { return readFile('file-list-changed.log') =~ /(?i)(^|\n)(salt\/.*\.py|setup\.py)\n/ }
}
steps {
sh '''
eval "$(pyenv init - --no-rehash)"
grep -Ei '^salt/.*\\.py$|^setup\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' tox -e pylint-salt | tee pylint-report-salt.log
# tee makes the exit/return code always 0
grep -Ei '^salt/.*\\.py$|^setup\\.py$' file-list-changed.log | (xargs -r '--delimiter=\\n' tox -e pylint-salt ; echo "$?" > pylint-salt-chg.exit) | tee pylint-report-salt-chg.log
# remove color escape coding
sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-salt.log
sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-salt-chg.log
read rc_exit < pylint-salt-chg.exit
exit "$rc_exit"
'''
archiveArtifacts artifacts: 'pylint-report-salt.log'
archiveArtifacts artifacts: 'pylint-report-salt-chg.log'
}
}
stage('test linting') {
stage('lint test chg') {
when {
expression { return readFile('file-list-changed.log') =~ /(?i)(^|\n)tests\/.*\.py\n/ }
}
steps {
sh '''
eval "$(pyenv init - --no-rehash)"
grep -Ei '^tests/.*\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' tox -e pylint-tests | tee pylint-report-tests.log
# tee makes the exit/return code always 0
grep -Ei '^tests/.*\\.py$' file-list-changed.log | (xargs -r '--delimiter=\\n' tox -e pylint-tests ; echo "$?" > pylint-tests-chg.exit) | tee pylint-report-tests-chg.log
# remove color escape coding
sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-tests.log
sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-tests-chg.log
read rc_exit < pylint-tests-chg.exit
exit "$rc_exit"
'''
archiveArtifacts artifacts: 'pylint-report-tests.log'
archiveArtifacts artifacts: 'pylint-report-tests-chg.log'
}
}
}
post {
always {
step([$class: 'WarningsPublisher',
parserConfigurations: [[
parserName: 'PyLint',
pattern: 'pylint-report*chg.log'
]],
failedTotalAll: '0',
useDeltaValues: false,
canRunOnFailed: true,
usePreviousBuildAsReference: true
])
}
}
}
stage('linting all') {
// perform a full linit if this is a merge forward and the change only lint passed.
when {
expression { return params.BRANCH_NAME =~ /(?i)^merge-/ }
}
parallel {
stage('setup full') {
steps {
githubNotify credentialsId: 'test-jenkins-credentials',
description: 'Python lint on everything begins...',
status: 'PENDING',
context: "jenkins/pr/lint"
}
}
stage('lint salt full') {
steps {
sh '''
eval "$(pyenv init - --no-rehash)"
(tox -e pylint-salt ; echo "$?" > pylint-salt-full.exit) | tee pylint-report-salt-full.log
# remove color escape coding
sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-salt-full.log
read rc_exit < pylint-salt-full.exit
exit "$rc_exit"
'''
archiveArtifacts artifacts: 'pylint-report-salt-full.log'
}
}
stage('lint test full') {
steps {
sh '''
eval "$(pyenv init - --no-rehash)"
(tox -e pylint-tests ; echo "$?" > pylint-tests-full.exit) | tee pylint-report-tests-full.log
# remove color escape coding
sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-tests-full.log
read rc_exit < pylint-tests-full.exit
exit "$rc_exit"
'''
archiveArtifacts artifacts: 'pylint-report-tests-full.log'
}
}
}
post {
always {
step([$class: 'WarningsPublisher',
parserConfigurations: [[
parserName: 'PyLint',
pattern: 'pylint-report*full.log'
]],
failedTotalAll: '0',
useDeltaValues: false,
canRunOnFailed: true,
usePreviousBuildAsReference: true
])
}
}
}
}
post {
always {
step([$class: 'WarningsPublisher',
parserConfigurations: [[
parserName: 'PyLint',
pattern: 'pylint-report*.log'
]],
failedTotalAll: '0',
useDeltaValues: false,
canRunOnFailed: true,
usePreviousBuildAsReference: true
])
cleanWs()
}
success {
githubNotify credentialsId: 'test-jenkins-credentials',
description: 'The lint job has passed',
description: 'Python lint test has passed',
status: 'SUCCESS',
context: "jenkins/pr/lint"
}
failure {
githubNotify credentialsId: 'test-jenkins-credentials',
description: 'The lint job has failed',
description: 'Python lint test has failed',
status: 'FAILURE',
context: "jenkins/pr/lint"
slackSend channel: "#jenkins-prod-pr",

4
.github/CODEOWNERS vendored
View file

@ -13,6 +13,7 @@ salt/*/*boto* @saltstack/team-boto
# Team Core
requirements/* @saltstack/team-core
rfcs/* @saltstack/team-core
salt/auth/* @saltstack/team-core
salt/cache/* @saltstack/team-core
salt/cli/* @saltstack/team-core
@ -73,3 +74,6 @@ salt/modules/reg.py @saltstack/team-windows
salt/states/reg.py @saltstack/team-windows
tests/*/*win* @saltstack/team-windows
tests/*/test_reg.py @saltstack/team-windows
# Jenkins Integration
.ci/* @saltstack/saltstack-sre-team @saltstack/team-core

22
salt/loader.py
View file

@ -138,6 +138,18 @@ def static_loader(
return ret
def _format_entrypoint_target(ep):
'''
Makes a string describing the target of an EntryPoint object.
Base strongly on EntryPoint.__str__().
'''
s = ep.module_name
if ep.attrs:
s += ':' + '.'.join(ep.attrs)
return s
def _module_dirs(
opts,
ext_type,
@ -160,9 +172,13 @@ def _module_dirs(
ext_type_types.extend(opts[ext_type_dirs])
if HAS_PKG_RESOURCES and ext_type_dirs:
for entry_point in pkg_resources.iter_entry_points('salt.loader', ext_type_dirs):
loaded_entry_point = entry_point.load()
for path in loaded_entry_point():
ext_type_types.append(path)
try:
loaded_entry_point = entry_point.load()
for path in loaded_entry_point():
ext_type_types.append(path)
except Exception as exc:
log.error("Error getting module directories from %s: %s", _format_entrypoint_target(entry_point), exc)
log.debug("Full backtrace for module directories error", exc_info=True)
cli_module_dirs = []
# The dirs can be any module dir, or a in-tree _{ext_type} dir

4
salt/modules/file.py
View file

@ -2096,6 +2096,8 @@ def replace(path,
# Just search; bail as early as a match is found
if re.search(cpattern, r_data):
return True # `with` block handles file closure
else:
return False
else:
result, nrepl = re.subn(cpattern,
repl.replace('\\', '\\\\') if backslash_literal else repl,
@ -5653,6 +5655,7 @@ def list_backups(path, limit=None):
[files[x] for x in sorted(files, reverse=True)[:limit]]
)))
list_backup = salt.utils.alias_function(list_backups, 'list_backup')
@ -5825,6 +5828,7 @@ def delete_backup(path, backup_id):
return ret
remove_backup = salt.utils.alias_function(delete_backup, 'remove_backup')

30
salt/states/gem.py
View file

@ -16,6 +16,9 @@ you can specify what ruby version and gemset to target.
'''
from __future__ import absolute_import
import salt.utils
import re
import logging
log = logging.getLogger(__name__)
@ -84,10 +87,29 @@ def installed(name, # pylint: disable=C0103
'Use of argument ruby found, but neither rvm or rbenv is installed'
)
gems = __salt__['gem.list'](name, ruby, gem_bin=gem_bin, runas=user)
if name in gems and version is not None and str(version) in gems[name]:
ret['result'] = True
ret['comment'] = 'Gem is already installed.'
return ret
if name in gems and version is not None:
match = re.match(r'(>=|>|<|<=)', version)
if match:
# Grab the comparison
cmpr = match.group()
# Clear out 'default:' and any whitespace
installed_version = re.sub('default: ', '', gems[name][0]).strip()
# Clear out comparison from version and whitespace
desired_version = re.sub(cmpr, '', version).strip()
if salt.utils.compare_versions(installed_version,
cmpr,
desired_version):
ret['result'] = True
ret['comment'] = 'Installed Gem meets version requirements.'
return ret
else:
if str(version) in gems[name]:
ret['result'] = True
ret['comment'] = 'Gem is already installed.'
return ret
elif name in gems and version is None:
ret['result'] = True
ret['comment'] = 'Gem is already installed.'

1
salt/states/supervisord.py
View file

@ -338,6 +338,7 @@ def dead(name,
else:
# process name doesn't exist
ret['comment'] = "Service {0} doesn't exist".format(name)
return ret
if is_stopped is True:
ret['comment'] = "Service {0} is not running".format(name)

16
tests/unit/modules/test_file.py
View file

@ -198,6 +198,22 @@ class FileReplaceTestCase(TestCase, LoaderModuleMockMixin):
'''
filemod.replace(self.tfile.name, r'Etiam', 123)
def test_search_only_return_true(self):
ret = filemod.replace(self.tfile.name,
r'Etiam', 'Salticus',
search_only=True)
self.assertIsInstance(ret, bool)
self.assertEqual(ret, True)
def test_search_only_return_false(self):
ret = filemod.replace(self.tfile.name,
r'Etian', 'Salticus',
search_only=True)
self.assertIsInstance(ret, bool)
self.assertEqual(ret, False)
class FileBlockReplaceTestCase(TestCase, LoaderModuleMockMixin):
def setup_loader_modules(self):

13
tests/unit/states/test_gem.py
View file

@ -47,6 +47,19 @@ class TestGemState(TestCase, LoaderModuleMockMixin):
ri=False, gem_bin=None
)
def test_installed_version(self):
gems = {'foo': ['1.0'], 'bar': ['2.0']}
gem_list = MagicMock(return_value=gems)
gem_install_succeeds = MagicMock(return_value=True)
with patch.dict(gem.__salt__, {'gem.list': gem_list}):
with patch.dict(gem.__salt__,
{'gem.install': gem_install_succeeds}):
ret = gem.installed('foo', version='>= 1.0')
self.assertEqual(True, ret['result'])
self.assertEqual('Installed Gem meets version requirements.',
ret['comment'])
def test_removed(self):
gems = ['foo', 'bar']
gem_list = MagicMock(return_value=gems)