From 2cc9fb14c838e37d7c54948a12ae96251198668f Mon Sep 17 00:00:00 2001 From: Joe Eacott Date: Fri, 19 Apr 2019 16:11:02 -0400 Subject: [PATCH 01/31] Corrected documentation to state default keep_job param is set to 24 and not 0. --- salt/returners/mysql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/returners/mysql.py b/salt/returners/mysql.py index c835c18eb1d..2479d0deb90 100644 --- a/salt/returners/mysql.py +++ b/salt/returners/mysql.py @@ -46,7 +46,8 @@ optional. The following ssl options are simply for illustration purposes: Should you wish the returner data to be cleaned out every so often, set `keep_jobs` to the number of hours for the jobs to live in the tables. -Setting it to `0` or leaving it unset will cause the data to stay in the tables. +Setting it to `0` will cause the data to stay in the tables. +The default setting for `keep_jobs` is set to `24`. Should you wish to archive jobs in a different table for later processing, set `archive_jobs` to True. Salt will create 3 archive tables From 4214b1cb238d55f828331e4047e41f233bf5743c Mon Sep 17 00:00:00 2001 From: Joe Eacott Date: Fri, 26 Apr 2019 15:10:29 -0400 Subject: [PATCH 02/31] Quick fix for tests. --- salt/returners/mysql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/returners/mysql.py b/salt/returners/mysql.py index 2479d0deb90..af41d26d325 100644 --- a/salt/returners/mysql.py +++ b/salt/returners/mysql.py @@ -46,8 +46,8 @@ optional. The following ssl options are simply for illustration purposes: Should you wish the returner data to be cleaned out every so often, set `keep_jobs` to the number of hours for the jobs to live in the tables. -Setting it to `0` will cause the data to stay in the tables. -The default setting for `keep_jobs` is set to `24`. +Setting it to `0` will cause the data to stay in the tables. The default +setting for `keep_jobs` is set to `24`. Should you wish to archive jobs in a different table for later processing, set `archive_jobs` to True. Salt will create 3 archive tables From 8a3267df162c8213838e23408d9bd78d3276e507 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 29 May 2019 11:28:55 +0100 Subject: [PATCH 03/31] Have sphinx turn errors into warnings --- noxfile.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index 819f70310fa..4994ff9bad5 100644 --- a/noxfile.py +++ b/noxfile.py @@ -893,9 +893,12 @@ def docs(session): ''' Build Salt's Documentation ''' - session.install('--progress-bar=off', '-r', 'requirements/static/py3.6/docs.txt', silent=PIP_INSTALL_SILENT) + session.install( + '--progress-bar=off', + '-r', 'requirements/static/py{}/docs.txt'.format(session.python), + silent=PIP_INSTALL_SILENT) os.chdir('doc/') session.run('make', 'clean', external=True) - session.run('make', 'html', external=True) + session.run('make', 'html', 'SPHINXOPTS=-W', external=True) session.run('tar', '-czvf', 'doc-archive.tar.gz', '_build/html') os.chdir('..') From a19d1f45b6a190fe59ff939074b0d83756be6760 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 29 May 2019 11:36:03 +0100 Subject: [PATCH 04/31] Allow docs to be built on Python >= 3.5, not just Python 3.6 --- .pre-commit-config.yaml | 18 ++++++++++++++++++ noxfile.py | 7 +++++-- requirements/static/py3.5/docs.txt | 30 ++++++++++++++++++++++++++++++ requirements/static/py3.7/docs.txt | 30 ++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 requirements/static/py3.5/docs.txt create mode 100644 requirements/static/py3.7/docs.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c684b048458..e40dd401fbe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -312,6 +312,15 @@ repos: - -v - --py-version=3.5 + - id: pip-tools-compile + alias: compile-doc-requirements + name: Docs Py3.5 Requirements + files: ^requirements/((base|zeromq|pytest)\.txt|static/docs\.in)$ + args: + - -v + - --py-version=3.5 + - --platform=linux + - id: pip-tools-compile alias: compile-linux-py3.6-tcp-requirements name: Linux Py3.6 TCP Requirements @@ -550,3 +559,12 @@ repos: args: - -v - --py-version=3.7 + + - id: pip-tools-compile + alias: compile-doc-requirements + name: Docs Py3.7 Requirements + files: ^requirements/((base|zeromq|pytest)\.txt|static/docs\.in)$ + args: + - -v + - --py-version=3.7 + - --platform=linux diff --git a/noxfile.py b/noxfile.py index 4994ff9bad5..27deca25813 100644 --- a/noxfile.py +++ b/noxfile.py @@ -888,14 +888,17 @@ def lint_tests(session): _lint(session, '.testing.pylintrc', flags, paths) -@nox.session(python='3.6') +@nox.session(python='3') def docs(session): ''' Build Salt's Documentation ''' + pydir = _get_pydir(session) + if pydir == 'py3.4': + session.error('Sphinx only runs on Python >= 3.5') session.install( '--progress-bar=off', - '-r', 'requirements/static/py{}/docs.txt'.format(session.python), + '-r', 'requirements/static/{}/docs.txt'.format(pydir), silent=PIP_INSTALL_SILENT) os.chdir('doc/') session.run('make', 'clean', external=True) diff --git a/requirements/static/py3.5/docs.txt b/requirements/static/py3.5/docs.txt new file mode 100644 index 00000000000..f159bfa46cd --- /dev/null +++ b/requirements/static/py3.5/docs.txt @@ -0,0 +1,30 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile -o requirements/static/py3.5/docs.txt -v requirements/static/docs.in +# +alabaster==0.7.12 # via sphinx +babel==2.7.0 # via sphinx +certifi==2019.3.9 # via requests +chardet==3.0.4 # via requests +docutils==0.14 # via sphinx +idna==2.8 # via requests +imagesize==1.1.0 # via sphinx +jinja2==2.10.1 # via sphinx +markupsafe==1.1.1 # via jinja2 +packaging==19.0 # via sphinx +pygments==2.4.2 # via sphinx +pyparsing==2.4.0 # via packaging +pytz==2019.1 # via babel +requests==2.22.0 # via sphinx +six==1.12.0 # via packaging +snowballstemmer==1.2.1 # via sphinx +sphinx==2.0.1 +sphinxcontrib-applehelp==1.0.1 # via sphinx +sphinxcontrib-devhelp==1.0.1 # via sphinx +sphinxcontrib-htmlhelp==1.0.2 # via sphinx +sphinxcontrib-jsmath==1.0.1 # via sphinx +sphinxcontrib-qthelp==1.0.2 # via sphinx +sphinxcontrib-serializinghtml==1.1.3 # via sphinx +urllib3==1.25.3 # via requests diff --git a/requirements/static/py3.7/docs.txt b/requirements/static/py3.7/docs.txt new file mode 100644 index 00000000000..917bdc06faa --- /dev/null +++ b/requirements/static/py3.7/docs.txt @@ -0,0 +1,30 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile -o requirements/static/py3.7/docs.txt -v requirements/static/docs.in +# +alabaster==0.7.12 # via sphinx +babel==2.7.0 # via sphinx +certifi==2019.3.9 # via requests +chardet==3.0.4 # via requests +docutils==0.14 # via sphinx +idna==2.8 # via requests +imagesize==1.1.0 # via sphinx +jinja2==2.10.1 # via sphinx +markupsafe==1.1.1 # via jinja2 +packaging==19.0 # via sphinx +pygments==2.4.2 # via sphinx +pyparsing==2.4.0 # via packaging +pytz==2019.1 # via babel +requests==2.22.0 # via sphinx +six==1.12.0 # via packaging +snowballstemmer==1.2.1 # via sphinx +sphinx==2.0.1 +sphinxcontrib-applehelp==1.0.1 # via sphinx +sphinxcontrib-devhelp==1.0.1 # via sphinx +sphinxcontrib-htmlhelp==1.0.2 # via sphinx +sphinxcontrib-jsmath==1.0.1 # via sphinx +sphinxcontrib-qthelp==1.0.2 # via sphinx +sphinxcontrib-serializinghtml==1.1.3 # via sphinx +urllib3==1.25.3 # via requests From 84665e2c74ce606ffa9484f03c3203840c09e8b3 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 4 Jun 2019 07:15:16 +0100 Subject: [PATCH 05/31] Fully revert 4eeedaa --- salt/log/setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/salt/log/setup.py b/salt/log/setup.py index c3d89cab99e..9cab84fad5e 100644 --- a/salt/log/setup.py +++ b/salt/log/setup.py @@ -1067,13 +1067,12 @@ def __process_multiprocessing_logging_queue(opts, queue): setup_extended_logging(opts) while True: try: - record_dict = queue.get() - if record_dict is None: + record = queue.get() + if record is None: # A sentinel to stop processing the queue break # Just log everything, filtering will happen on the main process # logging handlers - record = logging.makeLogRecord(record_dict) logger = logging.getLogger(record.name) logger.handle(record) except (EOFError, KeyboardInterrupt, SystemExit): From 06f295b43bb96c9e7a0a85d80ffa68c0b01429da Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 28 May 2019 11:09:58 +0100 Subject: [PATCH 06/31] Add the missing, and required, top pillar file. --- tests/integration/minion/test_blackout.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/integration/minion/test_blackout.py b/tests/integration/minion/test_blackout.py index 10b5bd3f184..6fd135743cf 100644 --- a/tests/integration/minion/test_blackout.py +++ b/tests/integration/minion/test_blackout.py @@ -28,15 +28,25 @@ class MinionBlackoutTestCase(ModuleCase): @classmethod def setUpClass(cls): + cls.top_pillar = os.path.join(RUNTIME_VARS.TMP_PILLAR_TREE, 'top.sls') cls.blackout_pillar = os.path.join(RUNTIME_VARS.TMP_PILLAR_TREE, 'blackout.sls') @classmethod def tearDownClass(cls): + if os.path.exists(cls.top_pillar): + os.unlink(cls.top_pillar) + del cls.top_pillar if os.path.exists(cls.blackout_pillar): os.unlink(cls.blackout_pillar) del cls.blackout_pillar def setUp(self): + with salt.utils.files.fopen(self.top_pillar, 'w') as wfh: + wfh.write(textwrap.dedent('''\ + base: + '*': + - blackout + ''')) with salt.utils.files.fopen(self.blackout_pillar, 'w') as wfh: wfh.write('minion_blackout: False') self.addCleanup(self.cleanup_blackout_pillar) @@ -49,6 +59,8 @@ class MinionBlackoutTestCase(ModuleCase): self.wait_for_all_jobs() def cleanup_blackout_pillar(self): + if os.path.exists(self.top_pillar): + os.unlink(self.top_pillar) if os.path.exists(self.blackout_pillar): os.unlink(self.blackout_pillar) From 5a256483d150996d21965da2afa6b3c997045d41 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 31 May 2019 17:39:51 +0100 Subject: [PATCH 07/31] Run the full test suite by default on 2018.3 --- .ci/kitchen-centos6-py2 | 2 +- .ci/kitchen-centos7-py2 | 2 +- .ci/kitchen-centos7-py3 | 2 +- .ci/kitchen-debian8-py2 | 2 +- .ci/kitchen-debian8-py3 | 2 +- .ci/kitchen-debian9-py2 | 2 +- .ci/kitchen-debian9-py3 | 2 +- .ci/kitchen-fedora28-py2 | 2 +- .ci/kitchen-fedora28-py3 | 2 +- .ci/kitchen-ubuntu1604-py2 | 2 +- .ci/kitchen-ubuntu1604-py3 | 2 +- .ci/kitchen-ubuntu1804-py2 | 2 +- .ci/kitchen-ubuntu1804-py3 | 2 +- .ci/kitchen-windows2016-py2 | 2 +- .ci/kitchen-windows2016-py3 | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.ci/kitchen-centos6-py2 b/.ci/kitchen-centos6-py2 index 011a6dd69cd..123974ef281 100644 --- a/.ci/kitchen-centos6-py2 +++ b/.ci/kitchen-centos6-py2 @@ -12,7 +12,7 @@ properties([ $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false ], parameters([ - booleanParam(defaultValue: false, description: 'Run full test suite', name: 'runFull') + booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) timeout(time: global_timeout, unit: 'HOURS') { diff --git a/.ci/kitchen-centos7-py2 b/.ci/kitchen-centos7-py2 index af6ed0754c6..02806f38d38 100644 --- a/.ci/kitchen-centos7-py2 +++ b/.ci/kitchen-centos7-py2 @@ -12,7 +12,7 @@ properties([ $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false ], parameters([ - booleanParam(defaultValue: false, description: 'Run full test suite', name: 'runFull') + booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) timeout(time: global_timeout, unit: 'HOURS') { diff --git a/.ci/kitchen-centos7-py3 b/.ci/kitchen-centos7-py3 index 959807d179e..e596b5e49c6 100644 --- a/.ci/kitchen-centos7-py3 +++ b/.ci/kitchen-centos7-py3 @@ -12,7 +12,7 @@ properties([ $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false ], parameters([ - booleanParam(defaultValue: false, description: 'Run full test suite', name: 'runFull') + booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) timeout(time: global_timeout, unit: 'HOURS') { diff --git a/.ci/kitchen-debian8-py2 b/.ci/kitchen-debian8-py2 index 4954ac44b64..8076c2de1f5 100644 --- a/.ci/kitchen-debian8-py2 +++ b/.ci/kitchen-debian8-py2 @@ -12,7 +12,7 @@ properties([ $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false ], parameters([ - booleanParam(defaultValue: false, description: 'Run full test suite', name: 'runFull') + booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) timeout(time: global_timeout, unit: 'HOURS') { diff --git a/.ci/kitchen-debian8-py3 b/.ci/kitchen-debian8-py3 index b4b674301f0..b645b10a9b2 100644 --- a/.ci/kitchen-debian8-py3 +++ b/.ci/kitchen-debian8-py3 @@ -12,7 +12,7 @@ properties([ $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false ], parameters([ - booleanParam(defaultValue: false, description: 'Run full test suite', name: 'runFull') + booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) timeout(time: global_timeout, unit: 'HOURS') { diff --git a/.ci/kitchen-debian9-py2 b/.ci/kitchen-debian9-py2 index 1e9d220c6de..e0557872461 100644 --- a/.ci/kitchen-debian9-py2 +++ b/.ci/kitchen-debian9-py2 @@ -12,7 +12,7 @@ properties([ $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false ], parameters([ - booleanParam(defaultValue: false, description: 'Run full test suite', name: 'runFull') + booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) timeout(time: global_timeout, unit: 'HOURS') { diff --git a/.ci/kitchen-debian9-py3 b/.ci/kitchen-debian9-py3 index cae46059b3f..53b6ef3cd2b 100644 --- a/.ci/kitchen-debian9-py3 +++ b/.ci/kitchen-debian9-py3 @@ -12,7 +12,7 @@ properties([ $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false ], parameters([ - booleanParam(defaultValue: false, description: 'Run full test suite', name: 'runFull') + booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) timeout(time: global_timeout, unit: 'HOURS') { diff --git a/.ci/kitchen-fedora28-py2 b/.ci/kitchen-fedora28-py2 index 5d4cbd109a9..0957fcb8a45 100644 --- a/.ci/kitchen-fedora28-py2 +++ b/.ci/kitchen-fedora28-py2 @@ -12,7 +12,7 @@ properties([ $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false ], parameters([ - booleanParam(defaultValue: false, description: 'Run full test suite', name: 'runFull') + booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) timeout(time: global_timeout, unit: 'HOURS') { diff --git a/.ci/kitchen-fedora28-py3 b/.ci/kitchen-fedora28-py3 index 2151d38868f..35444ab0d54 100644 --- a/.ci/kitchen-fedora28-py3 +++ b/.ci/kitchen-fedora28-py3 @@ -12,7 +12,7 @@ properties([ $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false ], parameters([ - booleanParam(defaultValue: false, description: 'Run full test suite', name: 'runFull') + booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) timeout(time: global_timeout, unit: 'HOURS') { diff --git a/.ci/kitchen-ubuntu1604-py2 b/.ci/kitchen-ubuntu1604-py2 index b41c178833d..11f3347af5b 100644 --- a/.ci/kitchen-ubuntu1604-py2 +++ b/.ci/kitchen-ubuntu1604-py2 @@ -12,7 +12,7 @@ properties([ $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false ], parameters([ - booleanParam(defaultValue: false, description: 'Run full test suite', name: 'runFull') + booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) timeout(time: global_timeout, unit: 'HOURS') { diff --git a/.ci/kitchen-ubuntu1604-py3 b/.ci/kitchen-ubuntu1604-py3 index 8d4f1e35893..f9f264ec9a4 100644 --- a/.ci/kitchen-ubuntu1604-py3 +++ b/.ci/kitchen-ubuntu1604-py3 @@ -12,7 +12,7 @@ properties([ $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false ], parameters([ - booleanParam(defaultValue: false, description: 'Run full test suite', name: 'runFull') + booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) timeout(time: global_timeout, unit: 'HOURS') { diff --git a/.ci/kitchen-ubuntu1804-py2 b/.ci/kitchen-ubuntu1804-py2 index ad88d4fb998..e8a3d36b286 100644 --- a/.ci/kitchen-ubuntu1804-py2 +++ b/.ci/kitchen-ubuntu1804-py2 @@ -12,7 +12,7 @@ properties([ $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false ], parameters([ - booleanParam(defaultValue: false, description: 'Run full test suite', name: 'runFull') + booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) timeout(time: global_timeout, unit: 'HOURS') { diff --git a/.ci/kitchen-ubuntu1804-py3 b/.ci/kitchen-ubuntu1804-py3 index c96ebd135df..899e854cb4b 100644 --- a/.ci/kitchen-ubuntu1804-py3 +++ b/.ci/kitchen-ubuntu1804-py3 @@ -12,7 +12,7 @@ properties([ $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false ], parameters([ - booleanParam(defaultValue: false, description: 'Run full test suite', name: 'runFull') + booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) timeout(time: global_timeout, unit: 'HOURS') { diff --git a/.ci/kitchen-windows2016-py2 b/.ci/kitchen-windows2016-py2 index 81561e8125a..8b4a61ddd98 100644 --- a/.ci/kitchen-windows2016-py2 +++ b/.ci/kitchen-windows2016-py2 @@ -12,7 +12,7 @@ properties([ $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false ], parameters([ - booleanParam(defaultValue: false, description: 'Run full test suite', name: 'runFull') + booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) timeout(time: global_timeout, unit: 'HOURS') { diff --git a/.ci/kitchen-windows2016-py3 b/.ci/kitchen-windows2016-py3 index c1aa75b6b5f..4167bfbc480 100644 --- a/.ci/kitchen-windows2016-py3 +++ b/.ci/kitchen-windows2016-py3 @@ -12,7 +12,7 @@ properties([ $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false ], parameters([ - booleanParam(defaultValue: false, description: 'Run full test suite', name: 'runFull') + booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) timeout(time: global_timeout, unit: 'HOURS') { From 144ec0401bea682e0066c4357bbf60810948ef61 Mon Sep 17 00:00:00 2001 From: cmcmarrow Date: Fri, 7 Jun 2019 14:05:45 +1100 Subject: [PATCH 08/31] adds dunder del exception --- salt/client/__init__.py | 2 ++ salt/daemons/flo/worker.py | 2 ++ salt/master.py | 2 ++ salt/minion.py | 4 ++++ salt/payload.py | 2 ++ salt/transport/ipc.py | 8 ++++++++ salt/transport/tcp.py | 16 ++++++++++++++++ salt/transport/zeromq.py | 8 ++++++++ salt/utils/asynchronous.py | 2 ++ salt/utils/event.py | 6 ++++++ salt/utils/rsax931.py | 4 ++++ salt/utils/vt.py | 2 ++ 12 files changed, 58 insertions(+) diff --git a/salt/client/__init__.py b/salt/client/__init__.py index 824bc773e6a..f8db9efbe6a 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -2004,6 +2004,7 @@ class LocalClient(object): raise tornado.gen.Return({'jid': payload['load']['jid'], 'minions': payload['load']['minions']}) + # pylint: disable=W1701 def __del__(self): # This IS really necessary! # When running tests, if self.events is not destroyed, we leak 2 @@ -2011,6 +2012,7 @@ class LocalClient(object): if hasattr(self, 'event'): # The call below will take care of calling 'self.event.destroy()' del self.event + # pylint: enable=W1701 def _clean_up_subscriptions(self, job_id): if self.opts.get('order_masters'): diff --git a/salt/daemons/flo/worker.py b/salt/daemons/flo/worker.py index 7975a0a1a01..98743b0c2ec 100644 --- a/salt/daemons/flo/worker.py +++ b/salt/daemons/flo/worker.py @@ -182,8 +182,10 @@ class SaltRaetWorkerSetup(ioflo.base.deeding.Deed): self.stack.value.transmit(init, self.stack.value.fetchUidByName(manor_yard.name)) self.stack.value.serviceAll() + # pylint: disable=W1701 def __del__(self): self.stack.server.close() + # pylint: enable=W1701 class SaltRaetWorkerRouter(ioflo.base.deeding.Deed): diff --git a/salt/master.py b/salt/master.py index 30983c30686..945214eb671 100644 --- a/salt/master.py +++ b/salt/master.py @@ -916,8 +916,10 @@ class ReqServer(salt.utils.process.SignalHandlingMultiprocessingProcess): self.process_manager.send_signal_to_processes(signum) self.process_manager.kill_children() + # pylint: disable=W1701 def __del__(self): self.destroy() + # pylint: enable=W1701 class MWorker(salt.utils.process.SignalHandlingMultiprocessingProcess): diff --git a/salt/minion.py b/salt/minion.py index 7a35b86abf8..5900ae1fc29 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -941,8 +941,10 @@ class MinionManager(MinionBase): self.process_manager = ProcessManager(name='MultiMinionProcessManager') self.io_loop.spawn_callback(self.process_manager.run, **{'asynchronous': True}) # Tornado backward compat + # pylint: disable=W1701 def __del__(self): self.destroy() + # pylint: enable=W1701 def _bind(self): # start up the event publisher, so we can see events during startup @@ -2742,8 +2744,10 @@ class Minion(MinionBase): for cb in six.itervalues(self.periodic_callbacks): cb.stop() + # pylint: disable=W1701 def __del__(self): self.destroy() + # pylint: enable=W1701 class Syndic(Minion): diff --git a/salt/payload.py b/salt/payload.py index c702b461e7b..0ec4d9a371a 100644 --- a/salt/payload.py +++ b/salt/payload.py @@ -400,5 +400,7 @@ class SREQ(object): if self.context.closed is False: self.context.term() + # pylint: disable=W1701 def __del__(self): self.destroy() + # pylint: enable=W1701 diff --git a/salt/transport/ipc.py b/salt/transport/ipc.py index 9c92fe2cb64..4924321d8b5 100644 --- a/salt/transport/ipc.py +++ b/salt/transport/ipc.py @@ -216,8 +216,10 @@ class IPCServer(object): if hasattr(self.sock, 'close'): self.sock.close() + # pylint: disable=W1701 def __del__(self): self.close() + # pylint: enable=W1701 class IPCClient(object): @@ -323,6 +325,7 @@ class IPCClient(object): yield tornado.gen.sleep(1) + # pylint: disable=W1701 def __del__(self): try: self.close() @@ -334,6 +337,7 @@ class IPCClient(object): # This is raised when Python's GC has collected objects which # would be needed when calling self.close() pass + # pylint: enable=W1701 def close(self): ''' @@ -548,8 +552,10 @@ class IPCMessagePublisher(object): if hasattr(self.sock, 'close'): self.sock.close() + # pylint: disable=W1701 def __del__(self): self.close() + # pylint: enable=W1701 class IPCMessageSubscriber(IPCClient): @@ -701,6 +707,8 @@ class IPCMessageSubscriber(IPCClient): if exc and not isinstance(exc, StreamClosedError): log.error("Read future returned exception %r", exc) + # pylint: disable=W1701 def __del__(self): if IPCMessageSubscriber in globals(): self.close() + # pylint: enable=W1701 diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py index 8e0d154c1ec..4d64c50ea8f 100644 --- a/salt/transport/tcp.py +++ b/salt/transport/tcp.py @@ -175,8 +175,10 @@ if USE_LOAD_BALANCER: self._socket.close() self._socket = None + # pylint: disable=W1701 def __del__(self): self.close() + # pylint: enable=W1701 def run(self): ''' @@ -288,8 +290,10 @@ class AsyncTCPReqChannel(salt.transport.client.ReqChannel): self._closing = True self.message_client.close() + # pylint: disable=W1701 def __del__(self): self.close() + # pylint: enable=W1701 def _package_load(self, load): return { @@ -395,8 +399,10 @@ class AsyncTCPPubChannel(salt.transport.mixins.auth.AESPubClientMixin, salt.tran if hasattr(self, 'message_client'): self.message_client.close() + # pylint: disable=W1701 def __del__(self): self.close() + # pylint: enable=W1701 def _package_load(self, load): return { @@ -574,8 +580,10 @@ class TCPReqServerChannel(salt.transport.mixins.auth.AESReqServerMixin, salt.tra except Exception as exc: log.exception('TCPReqServerChannel close generated an exception: %s', str(exc)) + # pylint: disable=W1701 def __del__(self): self.close() + # pylint: enable=W1701 def pre_fork(self, process_manager): ''' @@ -820,8 +828,10 @@ class SaltMessageClientPool(salt.transport.MessageClientPool): def __init__(self, opts, args=None, kwargs=None): super(SaltMessageClientPool, self).__init__(SaltMessageClient, opts, args=args, kwargs=kwargs) + # pylint: disable=W1701 def __del__(self): self.close() + # pylint: enable=W1701 def close(self): for message_client in self.message_clients: @@ -926,8 +936,10 @@ class SaltMessageClient(object): self.connect_callback = None self.disconnect_callback = None + # pylint: disable=W1701 def __del__(self): self.close() + # pylint: enable=W1701 def connect(self): ''' @@ -1167,8 +1179,10 @@ class Subscriber(object): # 'StreamClosedError' when the stream is closed. self._read_until_future.exception() + # pylint: disable=W1701 def __del__(self): self.close() + # pylint: enable=W1701 class PubServer(tornado.tcpserver.TCPServer, object): @@ -1207,8 +1221,10 @@ class PubServer(tornado.tcpserver.TCPServer, object): return self._closing = True + # pylint: disable=W1701 def __del__(self): self.close() + # pylint: enable=W1701 def _add_client_present(self, client): id_ = client.id_ diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index b4d5984a469..a0ce804b34a 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -204,6 +204,7 @@ class AsyncZeroMQReqChannel(salt.transport.client.ReqChannel): args=(self.opts, self.master_uri,), kwargs={'io_loop': self._io_loop}) + # pylint: disable=W1701 def __del__(self): ''' Since the message_client creates sockets and assigns them to the IOLoop we have to @@ -211,6 +212,7 @@ class AsyncZeroMQReqChannel(salt.transport.client.ReqChannel): ''' if hasattr(self, 'message_client'): self.message_client.destroy() + # pylint: enable=W1701 @property def master_uri(self): @@ -437,8 +439,10 @@ class AsyncZeroMQPubChannel(salt.transport.mixins.auth.AESPubClientMixin, salt.t if hasattr(self, 'context') and self.context.closed is False: self.context.term() + # pylint: disable=W1701 def __del__(self): self.destroy() + # pylint: enable=W1701 # TODO: this is the time to see if we are connected, maybe use the req channel to guess? @tornado.gen.coroutine @@ -989,8 +993,10 @@ class AsyncReqMessageClientPool(salt.transport.MessageClientPool): def __init__(self, opts, args=None, kwargs=None): super(AsyncReqMessageClientPool, self).__init__(AsyncReqMessageClient, opts, args=args, kwargs=kwargs) + # pylint: disable=W1701 def __del__(self): self.destroy() + # pylint: enable=W1701 def destroy(self): for message_client in self.message_clients: @@ -1060,8 +1066,10 @@ class AsyncReqMessageClient(object): if self.context.closed is False: self.context.term() + # pylint: disable=W1701 def __del__(self): self.destroy() + # pylint: enable=W1701 def _init_socket(self): if hasattr(self, 'stream'): diff --git a/salt/utils/asynchronous.py b/salt/utils/asynchronous.py index 4870c1710e7..e7663a6b23f 100644 --- a/salt/utils/asynchronous.py +++ b/salt/utils/asynchronous.py @@ -76,6 +76,7 @@ class SyncWrapper(object): self.io_loop.start() return future.result() + # pylint: disable=W1701 def __del__(self): ''' On deletion of the asynchronous wrapper, make sure to clean up the asynchronous stuff @@ -92,3 +93,4 @@ class SyncWrapper(object): elif hasattr(self, 'io_loop'): self.io_loop.close() del self.io_loop + # pylint: enable=W1701 diff --git a/salt/utils/event.py b/salt/utils/event.py index 6dfb7e790a9..0eb31bf765d 100644 --- a/salt/utils/event.py +++ b/salt/utils/event.py @@ -876,6 +876,7 @@ class SaltEvent(object): # This will handle reconnects return self.subscriber.read_async(event_handler) + # pylint: disable=W1701 def __del__(self): # skip exceptions in destroy-- since destroy() doesn't cover interpreter # shutdown-- where globals start going missing @@ -883,6 +884,7 @@ class SaltEvent(object): self.destroy() except Exception: pass + # pylint: enable=W1701 def __enter__(self): return self @@ -1071,8 +1073,10 @@ class AsyncEventPublisher(object): if hasattr(self, 'puller'): self.puller.close() + # pylint: disable=W1701 def __del__(self): self.close() + # pylint: enable=W1701 class EventPublisher(salt.utils.process.SignalHandlingMultiprocessingProcess): @@ -1173,8 +1177,10 @@ class EventPublisher(salt.utils.process.SignalHandlingMultiprocessingProcess): self.close() super(EventPublisher, self)._handle_signals(signum, sigframe) + # pylint: disable=W1701 def __del__(self): self.close() + # pylint: enable=W1701 class EventReturn(salt.utils.process.SignalHandlingMultiprocessingProcess): diff --git a/salt/utils/rsax931.py b/salt/utils/rsax931.py index aa1d33024ee..1ee72f49552 100644 --- a/salt/utils/rsax931.py +++ b/salt/utils/rsax931.py @@ -115,9 +115,11 @@ class RSAX931Signer(object): if not libcrypto.PEM_read_bio_RSAPrivateKey(self._bio, pointer(self._rsa), None, None): raise ValueError('invalid RSA private key') + # pylint: disable=W1701 def __del__(self): libcrypto.BIO_free(self._bio) libcrypto.RSA_free(self._rsa) + # pylint: enable=W1701 def sign(self, msg): ''' @@ -153,9 +155,11 @@ class RSAX931Verifier(object): if not libcrypto.PEM_read_bio_RSA_PUBKEY(self._bio, pointer(self._rsa), None, None): raise ValueError('invalid RSA public key') + # pylint: disable=W1701 def __del__(self): libcrypto.BIO_free(self._bio) libcrypto.RSA_free(self._rsa) + # pylint: enable=W1701 def verify(self, signed): ''' diff --git a/salt/utils/vt.py b/salt/utils/vt.py index 22feb9ec91b..c771204062e 100644 --- a/salt/utils/vt.py +++ b/salt/utils/vt.py @@ -931,6 +931,7 @@ class Terminal(object): # <---- Linux Methods ---------------------------------------------------- # ----- Cleanup!!! ------------------------------------------------------> + # pylint: disable=W1701 def __del__(self, _maxsize=sys.maxsize, _active=_ACTIVE): # pylint: disable=W0102 # I've disabled W0102 above which is regarding a dangerous default # value of [] for _ACTIVE, though, this is how Python itself handles @@ -945,5 +946,6 @@ class Terminal(object): if self.isalive() and _ACTIVE is not None: # Child is still running, keep us alive until we can wait on it. _ACTIVE.append(self) + # pylint: enable=W1701 # <---- Cleanup!!! ------------------------------------------------------- # <---- Platform Specific Methods -------------------------------------------- From 6e7605830b077ea5849a117e18ceb23a2d327690 Mon Sep 17 00:00:00 2001 From: cmcmarrow Date: Fri, 7 Jun 2019 14:08:17 +1100 Subject: [PATCH 09/31] adds dunder del exception --- salt/utils/vt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/utils/vt.py b/salt/utils/vt.py index c771204062e..4578a7b1d5b 100644 --- a/salt/utils/vt.py +++ b/salt/utils/vt.py @@ -947,5 +947,6 @@ class Terminal(object): # Child is still running, keep us alive until we can wait on it. _ACTIVE.append(self) # pylint: enable=W1701 + # <---- Cleanup!!! ------------------------------------------------------- # <---- Platform Specific Methods -------------------------------------------- From 1487a899b4c53922d6617e7559105be8018d5629 Mon Sep 17 00:00:00 2001 From: Charles McMarrow <48689152+cmcmarrow@users.noreply.github.com> Date: Fri, 7 Jun 2019 07:12:41 +1100 Subject: [PATCH 10/31] Update vt.py --- salt/utils/vt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/utils/vt.py b/salt/utils/vt.py index 4578a7b1d5b..c771204062e 100644 --- a/salt/utils/vt.py +++ b/salt/utils/vt.py @@ -947,6 +947,5 @@ class Terminal(object): # Child is still running, keep us alive until we can wait on it. _ACTIVE.append(self) # pylint: enable=W1701 - # <---- Cleanup!!! ------------------------------------------------------- # <---- Platform Specific Methods -------------------------------------------- From 8cb842056ec8a632a0b346f56250a0fa02e159fa Mon Sep 17 00:00:00 2001 From: cmcmarrow Date: Fri, 7 Jun 2019 17:30:24 +1100 Subject: [PATCH 11/31] add rule to pylintc --- .pylintrc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 390a82c9985..e8c468e0aa9 100644 --- a/.pylintrc +++ b/.pylintrc @@ -26,7 +26,8 @@ load-plugins=saltpylint.pep8, saltpylint.smartup, saltpylint.minpyver, saltpylint.blacklist, - saltpylint.thirdparty + saltpylint.thirdparty, + saltpylint.dunder_del # Use multiple processes to speed up Pylint. # Don't bump this values on PyLint 1.4.0 - Know bug that ignores the passed --rcfile From 11b619027b53b5541c6eb9809c62661acd16963b Mon Sep 17 00:00:00 2001 From: cmcmarrow Date: Sat, 8 Jun 2019 11:05:23 +1100 Subject: [PATCH 12/31] adds SaltPyLint>=v2019.6.4 --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index cd666765c53..fa55419bb92 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,7 +1,7 @@ -r base.txt mock>=2.0.0 -SaltPyLint>=v2017.3.6 +SaltPyLint>=v2019.6.4 testinfra>=1.7.0,!=1.17.0 # httpretty Needs to be here for now even though it's a dependency of boto. From a5e59358caedb8b8f0eea11f4b8dbb2ae1c356d7 Mon Sep 17 00:00:00 2001 From: Bryce Larson Date: Wed, 5 Jun 2019 14:30:35 -0600 Subject: [PATCH 13/31] only keep last 10 builds of every pr on jenkins to reduce disk usage --- .ci/docs | 1 + .ci/kitchen-centos6-py2 | 1 + .ci/kitchen-centos7-py2 | 1 + .ci/kitchen-centos7-py3 | 1 + .ci/kitchen-debian8-py2 | 1 + .ci/kitchen-debian8-py3 | 1 + .ci/kitchen-debian9-py2 | 1 + .ci/kitchen-debian9-py3 | 1 + .ci/kitchen-fedora28-py2 | 1 + .ci/kitchen-fedora28-py3 | 1 + .ci/kitchen-ubuntu1604-py2 | 1 + .ci/kitchen-ubuntu1604-py3 | 1 + .ci/kitchen-ubuntu1804-py2 | 1 + .ci/kitchen-ubuntu1804-py3 | 1 + .ci/kitchen-windows2016-py2 | 1 + .ci/kitchen-windows2016-py3 | 1 + .ci/lint | 1 + 17 files changed, 17 insertions(+) diff --git a/.ci/docs b/.ci/docs index dacbe17b261..c76fc4ef8bf 100644 --- a/.ci/docs +++ b/.ci/docs @@ -6,6 +6,7 @@ pipeline { timestamps() ansiColor('xterm') timeout(time: 2, unit: 'HOURS') + buildDiscarder(logRotator(numToKeepStr: '10')) } environment { PYENV_ROOT = "/usr/local/pyenv" diff --git a/.ci/kitchen-centos6-py2 b/.ci/kitchen-centos6-py2 index 123974ef281..7332a8adf81 100644 --- a/.ci/kitchen-centos6-py2 +++ b/.ci/kitchen-centos6-py2 @@ -5,6 +5,7 @@ def testrun_timeout = 6 def global_timeout = testrun_timeout + 1; properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ $class: 'ScannerJobProperty', doNotScan: false ], diff --git a/.ci/kitchen-centos7-py2 b/.ci/kitchen-centos7-py2 index 02806f38d38..d01f6484365 100644 --- a/.ci/kitchen-centos7-py2 +++ b/.ci/kitchen-centos7-py2 @@ -5,6 +5,7 @@ def testrun_timeout = 6 def global_timeout = testrun_timeout + 1; properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ $class: 'ScannerJobProperty', doNotScan: false ], diff --git a/.ci/kitchen-centos7-py3 b/.ci/kitchen-centos7-py3 index e596b5e49c6..f4e8f0a37f8 100644 --- a/.ci/kitchen-centos7-py3 +++ b/.ci/kitchen-centos7-py3 @@ -5,6 +5,7 @@ def testrun_timeout = 6 def global_timeout = testrun_timeout + 1; properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ $class: 'ScannerJobProperty', doNotScan: false ], diff --git a/.ci/kitchen-debian8-py2 b/.ci/kitchen-debian8-py2 index 8076c2de1f5..f2c1820503c 100644 --- a/.ci/kitchen-debian8-py2 +++ b/.ci/kitchen-debian8-py2 @@ -5,6 +5,7 @@ def testrun_timeout = 6 def global_timeout = testrun_timeout + 1; properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ $class: 'ScannerJobProperty', doNotScan: false ], diff --git a/.ci/kitchen-debian8-py3 b/.ci/kitchen-debian8-py3 index b645b10a9b2..b65120d6c56 100644 --- a/.ci/kitchen-debian8-py3 +++ b/.ci/kitchen-debian8-py3 @@ -5,6 +5,7 @@ def testrun_timeout = 6 def global_timeout = testrun_timeout + 1; properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ $class: 'ScannerJobProperty', doNotScan: false ], diff --git a/.ci/kitchen-debian9-py2 b/.ci/kitchen-debian9-py2 index e0557872461..047c4c8f08d 100644 --- a/.ci/kitchen-debian9-py2 +++ b/.ci/kitchen-debian9-py2 @@ -5,6 +5,7 @@ def testrun_timeout = 6 def global_timeout = testrun_timeout + 1; properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ $class: 'ScannerJobProperty', doNotScan: false ], diff --git a/.ci/kitchen-debian9-py3 b/.ci/kitchen-debian9-py3 index 53b6ef3cd2b..bd328a6fdcf 100644 --- a/.ci/kitchen-debian9-py3 +++ b/.ci/kitchen-debian9-py3 @@ -5,6 +5,7 @@ def testrun_timeout = 6 def global_timeout = testrun_timeout + 1; properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ $class: 'ScannerJobProperty', doNotScan: false ], diff --git a/.ci/kitchen-fedora28-py2 b/.ci/kitchen-fedora28-py2 index 0957fcb8a45..037e8b776f4 100644 --- a/.ci/kitchen-fedora28-py2 +++ b/.ci/kitchen-fedora28-py2 @@ -5,6 +5,7 @@ def testrun_timeout = 6 def global_timeout = testrun_timeout + 1; properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ $class: 'ScannerJobProperty', doNotScan: false ], diff --git a/.ci/kitchen-fedora28-py3 b/.ci/kitchen-fedora28-py3 index 35444ab0d54..65e8bc32177 100644 --- a/.ci/kitchen-fedora28-py3 +++ b/.ci/kitchen-fedora28-py3 @@ -5,6 +5,7 @@ def testrun_timeout = 6 def global_timeout = testrun_timeout + 1; properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ $class: 'ScannerJobProperty', doNotScan: false ], diff --git a/.ci/kitchen-ubuntu1604-py2 b/.ci/kitchen-ubuntu1604-py2 index 11f3347af5b..809a99102b7 100644 --- a/.ci/kitchen-ubuntu1604-py2 +++ b/.ci/kitchen-ubuntu1604-py2 @@ -5,6 +5,7 @@ def testrun_timeout = 6 def global_timeout = testrun_timeout + 1; properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ $class: 'ScannerJobProperty', doNotScan: false ], diff --git a/.ci/kitchen-ubuntu1604-py3 b/.ci/kitchen-ubuntu1604-py3 index f9f264ec9a4..d4d6477fb3f 100644 --- a/.ci/kitchen-ubuntu1604-py3 +++ b/.ci/kitchen-ubuntu1604-py3 @@ -5,6 +5,7 @@ def testrun_timeout = 6 def global_timeout = testrun_timeout + 1; properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ $class: 'ScannerJobProperty', doNotScan: false ], diff --git a/.ci/kitchen-ubuntu1804-py2 b/.ci/kitchen-ubuntu1804-py2 index e8a3d36b286..aa9e2732001 100644 --- a/.ci/kitchen-ubuntu1804-py2 +++ b/.ci/kitchen-ubuntu1804-py2 @@ -5,6 +5,7 @@ def testrun_timeout = 6 def global_timeout = testrun_timeout + 1; properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ $class: 'ScannerJobProperty', doNotScan: false ], diff --git a/.ci/kitchen-ubuntu1804-py3 b/.ci/kitchen-ubuntu1804-py3 index 899e854cb4b..b24c64e8c40 100644 --- a/.ci/kitchen-ubuntu1804-py3 +++ b/.ci/kitchen-ubuntu1804-py3 @@ -5,6 +5,7 @@ def testrun_timeout = 6 def global_timeout = testrun_timeout + 1; properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ $class: 'ScannerJobProperty', doNotScan: false ], diff --git a/.ci/kitchen-windows2016-py2 b/.ci/kitchen-windows2016-py2 index 8b4a61ddd98..7887314aa59 100644 --- a/.ci/kitchen-windows2016-py2 +++ b/.ci/kitchen-windows2016-py2 @@ -5,6 +5,7 @@ def testrun_timeout = 8 def global_timeout = testrun_timeout + 1; properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ $class: 'ScannerJobProperty', doNotScan: false ], diff --git a/.ci/kitchen-windows2016-py3 b/.ci/kitchen-windows2016-py3 index 4167bfbc480..6806c2b3058 100644 --- a/.ci/kitchen-windows2016-py3 +++ b/.ci/kitchen-windows2016-py3 @@ -5,6 +5,7 @@ def testrun_timeout = 8 def global_timeout = testrun_timeout + 1; properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ $class: 'ScannerJobProperty', doNotScan: false ], diff --git a/.ci/lint b/.ci/lint index 6967b8ee28a..7fdd4c3ab32 100644 --- a/.ci/lint +++ b/.ci/lint @@ -4,6 +4,7 @@ pipeline { timestamps() ansiColor('xterm') timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(numToKeepStr: '10')) } environment { PYENV_ROOT = "/usr/local/pyenv" From 1546e818f48bdd3203d4f349854367233f39022f Mon Sep 17 00:00:00 2001 From: Bryce Larson Date: Sun, 9 Jun 2019 16:29:45 -0600 Subject: [PATCH 14/31] only fetch pr target branch instead of all branches to save time and disk space --- .ci/kitchen-centos6-py2 | 1 + .ci/kitchen-centos7-py2 | 1 + .ci/kitchen-centos7-py3 | 1 + .ci/kitchen-debian8-py2 | 1 + .ci/kitchen-debian8-py3 | 1 + .ci/kitchen-debian9-py2 | 1 + .ci/kitchen-debian9-py3 | 1 + .ci/kitchen-fedora28-py2 | 1 + .ci/kitchen-fedora28-py3 | 1 + .ci/kitchen-ubuntu1604-py2 | 1 + .ci/kitchen-ubuntu1604-py3 | 1 + .ci/kitchen-ubuntu1804-py2 | 1 + .ci/kitchen-ubuntu1804-py3 | 1 + .ci/kitchen-windows2016-py2 | 1 + .ci/kitchen-windows2016-py3 | 1 + 15 files changed, 15 insertions(+) diff --git a/.ci/kitchen-centos6-py2 b/.ci/kitchen-centos6-py2 index 7332a8adf81..13fb59af362 100644 --- a/.ci/kitchen-centos6-py2 +++ b/.ci/kitchen-centos6-py2 @@ -51,6 +51,7 @@ timeout(time: global_timeout, unit: 'HOURS') { try { // Setup the kitchen required bundle stage('setup-bundle') { + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } try { diff --git a/.ci/kitchen-centos7-py2 b/.ci/kitchen-centos7-py2 index d01f6484365..88ac5de726f 100644 --- a/.ci/kitchen-centos7-py2 +++ b/.ci/kitchen-centos7-py2 @@ -51,6 +51,7 @@ timeout(time: global_timeout, unit: 'HOURS') { try { // Setup the kitchen required bundle stage('setup-bundle') { + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } try { diff --git a/.ci/kitchen-centos7-py3 b/.ci/kitchen-centos7-py3 index f4e8f0a37f8..80c82fdb5e0 100644 --- a/.ci/kitchen-centos7-py3 +++ b/.ci/kitchen-centos7-py3 @@ -51,6 +51,7 @@ timeout(time: global_timeout, unit: 'HOURS') { try { // Setup the kitchen required bundle stage('setup-bundle') { + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } try { diff --git a/.ci/kitchen-debian8-py2 b/.ci/kitchen-debian8-py2 index f2c1820503c..13b1e1ae6b2 100644 --- a/.ci/kitchen-debian8-py2 +++ b/.ci/kitchen-debian8-py2 @@ -51,6 +51,7 @@ timeout(time: global_timeout, unit: 'HOURS') { try { // Setup the kitchen required bundle stage('setup-bundle') { + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } try { diff --git a/.ci/kitchen-debian8-py3 b/.ci/kitchen-debian8-py3 index b65120d6c56..4999103ac41 100644 --- a/.ci/kitchen-debian8-py3 +++ b/.ci/kitchen-debian8-py3 @@ -51,6 +51,7 @@ timeout(time: global_timeout, unit: 'HOURS') { try { // Setup the kitchen required bundle stage('setup-bundle') { + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } try { diff --git a/.ci/kitchen-debian9-py2 b/.ci/kitchen-debian9-py2 index 047c4c8f08d..0b11c88725b 100644 --- a/.ci/kitchen-debian9-py2 +++ b/.ci/kitchen-debian9-py2 @@ -51,6 +51,7 @@ timeout(time: global_timeout, unit: 'HOURS') { try { // Setup the kitchen required bundle stage('setup-bundle') { + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } try { diff --git a/.ci/kitchen-debian9-py3 b/.ci/kitchen-debian9-py3 index bd328a6fdcf..4d47fef0c0a 100644 --- a/.ci/kitchen-debian9-py3 +++ b/.ci/kitchen-debian9-py3 @@ -51,6 +51,7 @@ timeout(time: global_timeout, unit: 'HOURS') { try { // Setup the kitchen required bundle stage('setup-bundle') { + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } try { diff --git a/.ci/kitchen-fedora28-py2 b/.ci/kitchen-fedora28-py2 index 037e8b776f4..0a46291394d 100644 --- a/.ci/kitchen-fedora28-py2 +++ b/.ci/kitchen-fedora28-py2 @@ -51,6 +51,7 @@ timeout(time: global_timeout, unit: 'HOURS') { try { // Setup the kitchen required bundle stage('setup-bundle') { + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } try { diff --git a/.ci/kitchen-fedora28-py3 b/.ci/kitchen-fedora28-py3 index 65e8bc32177..3d2565b9a64 100644 --- a/.ci/kitchen-fedora28-py3 +++ b/.ci/kitchen-fedora28-py3 @@ -51,6 +51,7 @@ timeout(time: global_timeout, unit: 'HOURS') { try { // Setup the kitchen required bundle stage('setup-bundle') { + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } try { diff --git a/.ci/kitchen-ubuntu1604-py2 b/.ci/kitchen-ubuntu1604-py2 index 809a99102b7..e3b7b6dcee9 100644 --- a/.ci/kitchen-ubuntu1604-py2 +++ b/.ci/kitchen-ubuntu1604-py2 @@ -51,6 +51,7 @@ timeout(time: global_timeout, unit: 'HOURS') { try { // Setup the kitchen required bundle stage('setup-bundle') { + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } try { diff --git a/.ci/kitchen-ubuntu1604-py3 b/.ci/kitchen-ubuntu1604-py3 index d4d6477fb3f..a8451473774 100644 --- a/.ci/kitchen-ubuntu1604-py3 +++ b/.ci/kitchen-ubuntu1604-py3 @@ -51,6 +51,7 @@ timeout(time: global_timeout, unit: 'HOURS') { try { // Setup the kitchen required bundle stage('setup-bundle') { + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } try { diff --git a/.ci/kitchen-ubuntu1804-py2 b/.ci/kitchen-ubuntu1804-py2 index aa9e2732001..6f397945d8b 100644 --- a/.ci/kitchen-ubuntu1804-py2 +++ b/.ci/kitchen-ubuntu1804-py2 @@ -51,6 +51,7 @@ timeout(time: global_timeout, unit: 'HOURS') { try { // Setup the kitchen required bundle stage('setup-bundle') { + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } try { diff --git a/.ci/kitchen-ubuntu1804-py3 b/.ci/kitchen-ubuntu1804-py3 index b24c64e8c40..eee089a5d99 100644 --- a/.ci/kitchen-ubuntu1804-py3 +++ b/.ci/kitchen-ubuntu1804-py3 @@ -51,6 +51,7 @@ timeout(time: global_timeout, unit: 'HOURS') { try { // Setup the kitchen required bundle stage('setup-bundle') { + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } try { diff --git a/.ci/kitchen-windows2016-py2 b/.ci/kitchen-windows2016-py2 index 7887314aa59..f2f42c423b2 100644 --- a/.ci/kitchen-windows2016-py2 +++ b/.ci/kitchen-windows2016-py2 @@ -50,6 +50,7 @@ timeout(time: global_timeout, unit: 'HOURS') { try { // Setup the kitchen required bundle stage('setup-bundle') { + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } try { diff --git a/.ci/kitchen-windows2016-py3 b/.ci/kitchen-windows2016-py3 index 6806c2b3058..73413e51076 100644 --- a/.ci/kitchen-windows2016-py3 +++ b/.ci/kitchen-windows2016-py3 @@ -50,6 +50,7 @@ timeout(time: global_timeout, unit: 'HOURS') { try { // Setup the kitchen required bundle stage('setup-bundle') { + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } try { From 55d3931d287b8f1ce406c597cada30ea74ead3e6 Mon Sep 17 00:00:00 2001 From: cmcmarrow Date: Tue, 11 Jun 2019 00:27:57 +1100 Subject: [PATCH 15/31] dunder del exception pylint --- requirements/dev.txt | 2 +- requirements/static/py2.7/lint.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index fa55419bb92..c4918e5f286 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,7 +1,7 @@ -r base.txt mock>=2.0.0 -SaltPyLint>=v2019.6.4 +SaltPyLint>=v2019.6.7 testinfra>=1.7.0,!=1.17.0 # httpretty Needs to be here for now even though it's a dependency of boto. diff --git a/requirements/static/py2.7/lint.txt b/requirements/static/py2.7/lint.txt index 078a544add4..bd432c1376f 100644 --- a/requirements/static/py2.7/lint.txt +++ b/requirements/static/py2.7/lint.txt @@ -14,6 +14,6 @@ mccabe==0.6.1 # via pylint modernize==0.5 # via saltpylint pycodestyle==2.5.0 # via saltpylint pylint==1.6.5 -saltpylint==2019.1.11 +saltpylint==2019.6.7 six==1.12.0 # via astroid, pylint wrapt==1.11.1 # via astroid From 47f133051562e116efaca357999f10aef641348a Mon Sep 17 00:00:00 2001 From: cmcmarrow Date: Tue, 11 Jun 2019 02:18:27 +1100 Subject: [PATCH 16/31] Dunder del exception pylint --- .testing.pylintrc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.testing.pylintrc b/.testing.pylintrc index 9557fd4937a..78e90c2a29c 100644 --- a/.testing.pylintrc +++ b/.testing.pylintrc @@ -23,7 +23,8 @@ load-plugins=saltpylint.pep8, saltpylint.smartup, saltpylint.minpyver, saltpylint.blacklist, - saltpylint.thirdparty + saltpylint.thirdparty, + saltpylint.dunder_del # Use multiple processes to speed up Pylint. # Don't bump this values on PyLint 1.4.0 - Know bug that ignores the passed --rcfile From b6d64f504a6249c16e427fffbf856efddd7a9532 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 10 Jun 2019 12:04:25 -0500 Subject: [PATCH 17/31] Fix traceback when making http query to non-listening host/port and raise_error=False (#53394) * Fix traceback when making http query to non-listening host/port The query result is None, which results in a traceback when we try to report the length of the response before and after decompressing. This fixes that by returning without attempting decompression if the response resolves as False. It also removes an extraneous elif which does nothing. * Add test case * Improve documentation for http.query remote-exec func Also, suppress full traceback when socket errors go uncaught and reraise as CommandExecutionError. * Add missing imports --- salt/modules/http.py | 16 ++++++++++++++-- salt/utils/http.py | 5 ++--- tests/unit/utils/test_http.py | 18 ++++++++++++++++++ 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/salt/modules/http.py b/salt/modules/http.py index 162460bc6ec..731130eb4b8 100644 --- a/salt/modules/http.py +++ b/salt/modules/http.py @@ -12,13 +12,22 @@ import time # Import Salt libs import salt.utils.http +from salt.exceptions import CommandExecutionError + +# Import 3rd-party libs +from salt.ext import six def query(url, **kwargs): ''' + .. versionadded:: 2015.5.0 + Query a resource, and decode the return data - .. versionadded:: 2015.5.0 + raise_error : True + If ``False``, and if a connection cannot be made, the error will be + suppressed and the body of the return will simply be ``None``. + CLI Example: @@ -35,7 +44,10 @@ def query(url, **kwargs): opts.update(kwargs['opts']) del kwargs['opts'] - return salt.utils.http.query(url=url, opts=opts, **kwargs) + try: + return salt.utils.http.query(url=url, opts=opts, **kwargs) + except Exception as exc: + raise CommandExecutionError(six.text_type(exc)) def wait_for_successful_query(url, wait_for=300, **kwargs): diff --git a/salt/utils/http.py b/salt/utils/http.py index 807b3e339b1..b187c566b50 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -104,6 +104,8 @@ def __decompressContent(coding, pgctnt): Currently supports identity/none, deflate, and gzip, which should cover 99%+ of the content on the internet. ''' + if not pgctnt: + return pgctnt log.trace("Decompressing %s byte content with compression type: %s", len(pgctnt), coding) @@ -122,9 +124,6 @@ def __decompressContent(coding, pgctnt): elif coding == "compress": raise ValueError("LZW compression is not currently supported") - elif coding == 'identity': - pass - log.trace("Content size after decompression: %s", len(pgctnt)) return pgctnt diff --git a/tests/unit/utils/test_http.py b/tests/unit/utils/test_http.py index ab40c2a1441..44880204b9e 100644 --- a/tests/unit/utils/test_http.py +++ b/tests/unit/utils/test_http.py @@ -5,6 +5,8 @@ # Import Salt Libs from __future__ import absolute_import, unicode_literals, print_function +import socket +from contextlib import closing # Import Salt Testing Libs from tests.support.unit import TestCase, skipIf @@ -90,3 +92,19 @@ class HTTPTestCase(TestCase): mock_ret = 'foo=XXXXXXXXXX&foo=XXXXXXXXXX&api_key=testing&' ret = http._sanitize_url_components(mock_component_list, 'foo') self.assertEqual(ret, mock_ret) + + def test_query_null_response(self): + ''' + This tests that we get a null response when raise_error=False and the + host/port cannot be reached. + ''' + host = '127.0.0.1' + + # Find unused port + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + sock.bind((host, 0)) + port = sock.getsockname()[1] + + url = 'http://{host}:{port}/'.format(host=host, port=port) + result = http.query(url, raise_error=False) + assert result == {'body': None}, result From 082919dcbd08d984995b38c03942ff7c7354e58c Mon Sep 17 00:00:00 2001 From: Bryce Larson Date: Mon, 10 Jun 2019 11:39:06 -0600 Subject: [PATCH 18/31] add fedora 29 tests to pr's --- .ci/{kitchen-fedora28-py2 => kitchen-fedora29-py2} | 4 ++-- .ci/{kitchen-fedora28-py3 => kitchen-fedora29-py3} | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename .ci/{kitchen-fedora28-py2 => kitchen-fedora29-py2} (98%) rename .ci/{kitchen-fedora28-py3 => kitchen-fedora29-py3} (98%) diff --git a/.ci/kitchen-fedora28-py2 b/.ci/kitchen-fedora29-py2 similarity index 98% rename from .ci/kitchen-fedora28-py2 rename to .ci/kitchen-fedora29-py2 index 0a46291394d..4b959bc8276 100644 --- a/.ci/kitchen-fedora28-py2 +++ b/.ci/kitchen-fedora29-py2 @@ -28,11 +28,11 @@ timeout(time: global_timeout, unit: 'HOURS') { 'NOX_PASSTHROUGH_OPTS=--ssh-tests', 'NOX_ENABLE_FROM_FILENAMES=true', 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=fedora28,py2', + 'CODECOV_FLAGS=fedora29,py2', 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', 'RBENV_VERSION=2.4.2', 'TEST_SUITE=py2', - 'TEST_PLATFORM=fedora-28', + 'TEST_PLATFORM=fedora-29', 'PY_COLORS=1', "FORCE_FULL=${params.runFull}", ]) { diff --git a/.ci/kitchen-fedora28-py3 b/.ci/kitchen-fedora29-py3 similarity index 98% rename from .ci/kitchen-fedora28-py3 rename to .ci/kitchen-fedora29-py3 index 3d2565b9a64..ad6079e7e8c 100644 --- a/.ci/kitchen-fedora28-py3 +++ b/.ci/kitchen-fedora29-py3 @@ -28,11 +28,11 @@ timeout(time: global_timeout, unit: 'HOURS') { 'NOX_PASSTHROUGH_OPTS=--ssh-tests', 'NOX_ENABLE_FROM_FILENAMES=true', 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=fedora28,py3', + 'CODECOV_FLAGS=fedora29,py3', 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', 'RBENV_VERSION=2.4.2', 'TEST_SUITE=py3', - 'TEST_PLATFORM=fedora-28', + 'TEST_PLATFORM=fedora-29', 'PY_COLORS=1', "FORCE_FULL=${params.runFull}", ]) { From 95037c2538527967f8ab2b49c486d2b5e8df8142 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 29 May 2019 12:09:47 +0100 Subject: [PATCH 19/31] Handle nox running under Py2 and Py3 --- noxfile.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 27deca25813..e53ab26dfde 100644 --- a/noxfile.py +++ b/noxfile.py @@ -25,6 +25,8 @@ if __name__ == '__main__': import nox from nox.command import CommandFailed +IS_PY3 = sys.version_info > (2,) + # Be verbose when runing under a CI context PIP_INSTALL_SILENT = (os.environ.get('JENKINS_URL') or os.environ.get('CI') or os.environ.get('DRONE')) is None @@ -837,8 +839,12 @@ def _lint(session, rcfile, flags, paths): raise finally: stdout.seek(0) - contents = stdout.read().encode('utf-8') + contents = stdout.read() if contents: + if IS_PY3: + contents = contents.decode('utf-8') + else: + contents = contents.encode('utf-8') sys.stdout.write(contents) sys.stdout.flush() if pylint_report_path: From 55b64f69b1a844e1513ee7db6737201fa58c66fd Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 3 Jun 2019 09:49:35 +0100 Subject: [PATCH 20/31] Stop compiling static TCP requirements. --- .pre-commit-config.yaml | 126 ---------------------------------------- 1 file changed, 126 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e40dd401fbe..fa5a57238b0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,40 +3,6 @@ repos: - repo: https://github.com/saltstack/pip-tools-compile-impersonate rev: master hooks: - - id: pip-tools-compile - alias: compile-linux-py2.7-tcp-requirements - name: Linux Py2.7 TCP Requirements - files: ^requirements/((base|pytest)\.txt|static/(.*)\.in)$ - exclude: ^requirements/static/(lint|cloud|docs|osx|windows|)\.in$ - args: - - -v - - --py-version=2.7 - - --platform=linux - - --out-prefix=tcp - - --include=requirements/base.txt - - --include=requirements/pytest.txt - - id: pip-tools-compile - alias: compile-osx-py2.7-tcp-requirements - name: OSX Py2.7 TCP Requirements - files: ^requirements/((base|pytest)\.txt|static/osx\.in)$ - args: - - -v - - --py-version=2.7 - - --platform=darwin - - --out-prefix=tcp - - --include=requirements/base.txt - - --include=requirements/pytest.txt - - id: pip-tools-compile - alias: compile-windows-py2.7-tcp-requirements - name: Windows Py2.7 TCP Requirements - files: ^requirements/((base|pytest)\.txt|static/windows\.in)$ - args: - - -v - - --py-version=2.7 - - --platform=windows - - --out-prefix=tcp - - --include=requirements/base.txt - - --include=requirements/pytest.txt - id: pip-tools-compile alias: compile-linux-py2.7-zmq-requirements name: Linux Py2.7 ZeroMQ Requirements @@ -136,18 +102,6 @@ repos: - --py-version=2.7 - - id: pip-tools-compile - alias: compile-linux-py3.4-tcp-requirements - name: Linux Py3.4 TCP Requirements - files: ^requirements/((base|pytest)\.txt|static/(.*)\.in)$ - exclude: ^requirements/static/(centos-6|ubuntu-14.04|lint|cloud|docs|osx|windows)\.in$ - args: - - -v - - --py-version=3.4 - - --platform=linux - - --out-prefix=tcp - - --include=requirements/base.txt - - --include=requirements/pytest.txt - id: pip-tools-compile alias: compile-linux-py3.4-zmq-requirements name: Linux Py3.4 ZeroMQ Requirements @@ -186,40 +140,6 @@ repos: - -v - --py-version=3.4 - - id: pip-tools-compile - alias: compile-linux-py3.5-tcp-requirements - name: Linux Py3.5 TCP Requirements - files: ^requirements/((base|pytest)\.txt|static/(.*)\.in)$ - exclude: ^requirements/static/(centos-6|ubuntu-14.04|lint|cloud|docs|osx|windows)\.in$ - args: - - -v - - --py-version=3.5 - - --platform=linux - - --out-prefix=tcp - - --include=requirements/base.txt - - --include=requirements/pytest.txt - - id: pip-tools-compile - alias: compile-osx-py3.5-tcp-requirements - name: OSX Py3.5 TCP Requirements - files: ^requirements/((base|pytest)\.txt|static/osx\.in)$ - args: - - -v - - --py-version=3.5 - - --platform=darwin - - --out-prefix=tcp - - --include=requirements/base.txt - - --include=requirements/pytest.txt - - id: pip-tools-compile - alias: compile-windows-py3.5-tcp-requirements - name: Windows Py3.5 TCP Requirements - files: ^requirements/((base|pytest)\.txt|static/windows\.in)$ - args: - - -v - - --py-version=3.5 - - --platform=windows - - --out-prefix=tcp - - --include=requirements/base.txt - - --include=requirements/pytest.txt - id: pip-tools-compile alias: compile-linux-py3.5-zmq-requirements name: Linux Py3.5 ZeroMQ Requirements @@ -321,29 +241,6 @@ repos: - --py-version=3.5 - --platform=linux - - id: pip-tools-compile - alias: compile-linux-py3.6-tcp-requirements - name: Linux Py3.6 TCP Requirements - files: ^requirements/((base|pytest)\.txt|static/(.*)\.in)$ - exclude: ^requirements/static/(centos-6|ubuntu-14.04|lint|cloud|docs|osx|windows)\.in$ - args: - - -v - - --py-version=3.6 - - --platform=linux - - --out-prefix=tcp - - --include=requirements/base.txt - - --include=requirements/pytest.txt - - id: pip-tools-compile - alias: compile-osx-py3.6-tcp-requirements - name: OSX Py3.6 TCP Requirements - files: ^requirements/((base|pytest)\.txt|static/osx\.in)$ - args: - - -v - - --py-version=3.6 - - --platform=darwin - - --out-prefix=tcp - - --include=requirements/base.txt - - --include=requirements/pytest.txt - id: pip-tools-compile alias: compile-linux-py3.6-zmq-requirements name: Linux Py3.6 ZeroMQ Requirements @@ -445,29 +342,6 @@ repos: - --py-version=3.6 - --platform=linux - - id: pip-tools-compile - alias: compile-linux-py3.7-tcp-requirements - name: Linux Py3.7 TCP Requirements - files: ^requirements/((base|pytest)\.txt|static/(.*)\.in)$ - exclude: ^requirements/static/(centos-6|ubuntu-14.04|lint|cloud|docs|osx|windows)\.in$ - args: - - -v - - --py-version=3.7 - - --platform=linux - - --out-prefix=tcp - - --include=requirements/base.txt - - --include=requirements/pytest.txt - - id: pip-tools-compile - alias: compile-osx-py3.7-tcp-requirements - name: OSX Py3.7 TCP Requirements - files: ^requirements/((base|pytest)\.txt|static/osx\.in)$ - args: - - -v - - --py-version=3.7 - - --platform=darwin - - --out-prefix=tcp - - --include=requirements/base.txt - - --include=requirements/pytest.txt - id: pip-tools-compile alias: compile-linux-py3.7-zmq-requirements name: Linux Py3.7 ZeroMQ Requirements From e983c8c292e75080b3bc66e38e396ce97f0d1baa Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 3 Jun 2019 09:58:45 +0100 Subject: [PATCH 21/31] Fix Amazon 2018.03 in excludes --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fa5a57238b0..260d235488a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -106,7 +106,7 @@ repos: alias: compile-linux-py3.4-zmq-requirements name: Linux Py3.4 ZeroMQ Requirements files: ^requirements/((base|zeromq|pytest)\.txt|static/(.*)\.in)$ - exclude: ^requirements/static/(centos-6|ubuntu-14.04|lint|cloud|docs|osx|windows)\.in$ + exclude: ^requirements/static/(centos-6|amzn-2018\.03|lint|cloud|docs|osx|windows)\.in$ args: - -v - --py-version=3.4 @@ -120,7 +120,7 @@ repos: alias: compile-linux-py3.4-raet-requirements name: Linux Py3.4 RAET Requirements files: ^requirements/((base|raet|pytest)\.txt|static/(.*)\.in)$ - exclude: ^requirements/static/(centos-6|ubuntu-14.04|lint|cloud|docs|osx|windows)\.in$ + exclude: ^requirements/static/(centos-6|amzn-2018\.03|lint|cloud|docs|osx|windows)\.in$ args: - -v - --py-version=3.4 @@ -144,7 +144,7 @@ repos: alias: compile-linux-py3.5-zmq-requirements name: Linux Py3.5 ZeroMQ Requirements files: ^requirements/((base|zeromq|pytest)\.txt|static/(.*)\.in)$ - exclude: ^requirements/static/(centos-6|ubuntu-14.04|lint|cloud|docs|osx|windows)\.in$ + exclude: ^requirements/static/(centos-6|amzn-2018\.03|lint|cloud|docs|osx|windows)\.in$ args: - -v - --py-version=3.5 @@ -184,7 +184,7 @@ repos: alias: compile-linux-py3.5-raet-requirements name: Linux Py3.5 RAET Requirements files: ^requirements/((base|raet|pytest)\.txt|static/(.*)\.in)$ - exclude: ^requirements/static/(centos-6|ubuntu-14.04|lint|cloud|docs|osx|windows)\.in$ + exclude: ^requirements/static/(centos-6|amzn-2018\.03|lint|cloud|docs|osx|windows)\.in$ args: - -v - --py-version=3.5 @@ -245,7 +245,7 @@ repos: alias: compile-linux-py3.6-zmq-requirements name: Linux Py3.6 ZeroMQ Requirements files: ^requirements/((base|zeromq|pytest)\.txt|static/(.*)\.in)$ - exclude: ^requirements/static/(centos-6|ubuntu-14.04|lint|cloud|docs|osx|windows)\.in$ + exclude: ^requirements/static/(centos-6|amzn-2018\.03|lint|cloud|docs|osx|windows)\.in$ args: - -v - --py-version=3.6 @@ -285,7 +285,7 @@ repos: alias: compile-linux-py3.6-raet-requirements name: Linux Py3.6 RAET Requirements files: ^requirements/((base|raet|pytest)\.txt|static/(.*)\.in)$ - exclude: ^requirements/static/(centos-6|ubuntu-14.04|lint|cloud|docs|osx|windows)\.in$ + exclude: ^requirements/static/(centos-6|amzn-2018\.03|lint|cloud|docs|osx|windows)\.in$ args: - -v - --py-version=3.6 @@ -346,7 +346,7 @@ repos: alias: compile-linux-py3.7-zmq-requirements name: Linux Py3.7 ZeroMQ Requirements files: ^requirements/((base|zeromq|pytest)\.txt|static/(.*)\.in)$ - exclude: ^requirements/static/(centos-6|ubuntu-14.04|lint|cloud|docs|osx|windows)\.in$ + exclude: ^requirements/static/(centos-6|amzn-2018\.03|lint|cloud|docs|osx|windows)\.in$ args: - -v - --py-version=3.7 @@ -386,7 +386,7 @@ repos: alias: compile-linux-py3.7-raet-requirements name: Linux Py3.7 RAET Requirements files: ^requirements/((base|raet|pytest)\.txt|static/(.*)\.in)$ - exclude: ^requirements/static/(centos-6|ubuntu-14.04|lint|cloud|docs|osx|windows)\.in$ + exclude: ^requirements/static/(centos-6|amzn-2018\.03|lint|cloud|docs|osx|windows)\.in$ args: - -v - --py-version=3.7 From e29e62e2d0b3fb3def551b2bbaeeca4ca788bad4 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 6 Jun 2019 11:26:24 +0100 Subject: [PATCH 22/31] Take into account the packaging requirements for windows --- .pre-commit-config.yaml | 32 +++++++++--- pkg/windows/req.txt | 45 +++++++++-------- requirements/static/py2.7/raet-windows.txt | 51 ++++++++++++++------ requirements/static/py2.7/zeromq-windows.txt | 50 +++++++++++++------ requirements/static/py3.5/raet-windows.txt | 49 ++++++++++++++----- requirements/static/py3.5/zeromq-windows.txt | 47 +++++++++++++----- requirements/static/py3.6/raet-windows.txt | 49 ++++++++++++++----- requirements/static/py3.6/zeromq-windows.txt | 47 +++++++++++++----- requirements/static/py3.7/raet-windows.txt | 51 ++++++++++++++------ requirements/static/py3.7/zeromq-windows.txt | 49 +++++++++++++------ 10 files changed, 330 insertions(+), 140 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 260d235488a..703998f79b7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,12 +33,14 @@ repos: - id: pip-tools-compile alias: compile-windows-py2.7-zmq-requirements name: Windows Py2.7 ZeroMQ Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/windows\.in)$ + files: ^(pkg/windows/(req|req_win)\.txt|requirements/((base|zeromq|pytest)\.txt|static/windows\.in))$ args: - -v - --py-version=2.7 - --platform=windows - --out-prefix=zeromq + - --include=pkg/windows/req.txt + - --include=pkg/windows/req_win.txt - --include=requirements/base.txt - --include=requirements/zeromq.txt - --include=requirements/pytest.txt @@ -73,12 +75,14 @@ repos: - id: pip-tools-compile alias: compile-windows-py2.7-raet-requirements name: Windows Py2.7 RAET Requirements - files: ^requirements/((base|raet|pytest)\.txt|static/windows\.in)$ + files: ^(pkg/windows/(req|req_win)\.txt|requirements/((base|raet|pytest)\.txt|static/windows\.in))$ args: - -v - --py-version=2.7 - --out-prefix=raet - --platform=windows + - --include=pkg/windows/req.txt + - --include=pkg/windows/req_win.txt - --include=requirements/base.txt - --include=requirements/raet.txt - --include=requirements/pytest.txt @@ -170,12 +174,14 @@ repos: - id: pip-tools-compile alias: compile-windows-py3.5-zmq-requirements name: Windows Py3.5 ZeroMQ Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/windows\.in)$ + files: ^(pkg/windows/(req|req_win)\.txt|requirements/((base|zeromq|pytest)\.txt|static/windows\.in))$ args: - -v - --py-version=3.5 - --platform=windows - --out-prefix=zeromq + - --include=pkg/windows/req.txt + - --include=pkg/windows/req_win.txt - --include=requirements/base.txt - --include=requirements/zeromq.txt - --include=requirements/pytest.txt @@ -212,12 +218,14 @@ repos: - id: pip-tools-compile alias: compile-windows-py3.5-raet-requirements name: Windows Py3.5 RAET Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/windows\.in)$ + files: ^(pkg/windows/(req|req_win)\.txt|requirements/((base|raet|pytest)\.txt|static/windows\.in))$ args: - -v - --py-version=3.5 - --platform=windows - --out-prefix=raet + - --include=pkg/windows/req.txt + - --include=pkg/windows/req_win.txt - --include=requirements/base.txt - --include=requirements/raet.txt - --include=requirements/pytest.txt @@ -271,12 +279,14 @@ repos: - id: pip-tools-compile alias: compile-windows-py3.6-zmq-requirements name: Windows Py3.6 ZeroMQ Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/windows\.in)$ + files: ^(pkg/windows/(req|req_win)\.txt|requirements/((base|zeromq|pytest)\.txt|static/windows\.in))$ args: - -v - --py-version=3.6 - --platform=windows - --out-prefix=zeromq + - --include=pkg/windows/req.txt + - --include=pkg/windows/req_win.txt - --include=requirements/base.txt - --include=requirements/zeromq.txt - --include=requirements/pytest.txt @@ -313,12 +323,14 @@ repos: - id: pip-tools-compile alias: compile-windows-py3.6-raet-requirements name: Windows Py3.6 RAET Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/windows\.in)$ + files: ^(pkg/windows/(req|req_win)\.txt|requirements/((base|raet|pytest)\.txt|static/windows\.in))$ args: - -v - --py-version=3.6 - --platform=windows - --out-prefix=raet + - --include=pkg/windows/req.txt + - --include=pkg/windows/req_win.txt - --include=requirements/base.txt - --include=requirements/raet.txt - --include=requirements/pytest.txt @@ -372,12 +384,14 @@ repos: - id: pip-tools-compile alias: compile-windows-py3.7-zmq-requirements name: Windows Py3.7 ZeroMQ Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/windows\.in)$ + files: ^(pkg/windows/(req|req_win)\.txt|requirements/((base|zeromq|pytest)\.txt|static/windows\.in))$ args: - -v - --py-version=3.7 - --platform=windows - --out-prefix=zeromq + - --include=pkg/windows/req.txt + - --include=pkg/windows/req_win.txt - --include=requirements/base.txt - --include=requirements/zeromq.txt - --include=requirements/pytest.txt @@ -414,12 +428,14 @@ repos: - id: pip-tools-compile alias: compile-windows-py3.7-raet-requirements name: Windows Py3.7 RAET Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/windows\.in)$ + files: ^(pkg/windows/(req|req_win)\.txt|requirements/((base|raet|pytest)\.txt|static/windows\.in))$ args: - -v - --py-version=3.7 - --platform=windows - --out-prefix=raet + - --include=pkg/windows/req.txt + - --include=pkg/windows/req_win.txt - --include=requirements/base.txt - --include=requirements/raet.txt - --include=requirements/pytest.txt diff --git a/pkg/windows/req.txt b/pkg/windows/req.txt index 24eef938521..9072d978143 100644 --- a/pkg/windows/req.txt +++ b/pkg/windows/req.txt @@ -1,39 +1,38 @@ -r req_win.txt backports-abc==0.5 -backports.ssl-match-hostname==3.5.0.1 -certifi -cffi==1.10.0 +backports.ssl-match-hostname==3.7.0.1 +certifi==2019.3.9 +cffi==1.12.2 CherryPy==10.2.1 -cryptography==2.1.4 +cryptography==2.6.1 enum34==1.1.6 -futures==3.1.1 +futures==3.2.0 ; python_version < "3.0" gitdb==0.6.4 GitPython==2.1.3 -idna==2.5 +idna==2.8 ioloop==0.1a0 -ipaddress==1.0.18 -Jinja2==2.9.6 -libnacl==1.6.1 # required by the nacl module +ipaddress==1.0.22 +jinja2==2.10.1 +libnacl==1.6.1 lxml==3.7.3 Mako==1.0.6 -MarkupSafe==1.0 -msgpack-python==0.4.8 -psutil==5.2.2 -pyasn1==0.2.3 -pycparser==2.17 -pycrypto==2.6.1 +markupsafe==1.1.1 +msgpack-python==0.5.6 +psutil==5.6.1 +pyasn1==0.4.5 +pycparser==2.19 +pycryptodomex==3.8.1 pycurl==7.43.0 PyMySQL==0.7.11 -pyOpenSSL==17.5.0 -#python-certifi-win32==1.2 -python-dateutil==2.6.1 -python-gnupg==0.4.1 +pyopenssl==19.0.0 +python-dateutil==2.8.0 +python-gnupg==0.4.4 pythonnet==2.3.0 -PyYAML==3.12 -pyzmq==16.0.3 +pyyaml==3.13 +pyzmq==18.0.1 requests==2.21.0 singledispatch==3.4.0.3 smmap==0.9.0 timelib==0.2.4 -tornado==4.5.1 -wheel==0.30.0a0 +tornado==4.5.3 +wheel==0.33.4 diff --git a/requirements/static/py2.7/raet-windows.txt b/requirements/static/py2.7/raet-windows.txt index f9acca02a13..91ab7a8ba39 100644 --- a/requirements/static/py2.7/raet-windows.txt +++ b/requirements/static/py2.7/raet-windows.txt @@ -2,42 +2,50 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/py2.7/raet-windows.txt -v requirements/base.txt requirements/raet.txt requirements/pytest.txt requirements/static/windows.in +# pip-compile -o requirements/static/py2.7/raet-windows.txt -v pkg/windows/req.txt pkg/windows/req_win.txt requirements/base.txt requirements/raet.txt requirements/pytest.txt requirements/static/windows.in # asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado -backports.ssl-match-hostname==3.7.0.1 # via docker, websocket-client +backports-abc==0.5 +backports.functools-lru-cache==1.5 # via cheroot, jaraco.functools +backports.ssl-match-hostname==3.7.0.1 backports.tempfile==1.0 # via moto backports.weakref==1.0.post1 # via backports.tempfile boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 # via kubernetes, requests, tornado -cffi==1.12.2 # via cryptography, pygit2 +certifi==2019.3.9 +cffi==1.12.2 chardet==3.0.4 # via requests +cheroot==6.5.5 # via cherrypy +cherrypy==10.2.1 colorama==0.4.1 # via pytest cookies==2.2.1 # via responses coverage==4.5.3 # via pytest-cov -cryptography==2.6.1 # via moto, pyopenssl +cryptography==2.6.1 dmidecode==0.9.0 dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==2.7.0 docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose -enum34==1.1.6 # via cryptography, raet +enum34==1.1.6 funcsigs==1.0.2 # via mock, pytest functools32==3.2.3.post2 # via jsonschema future==0.17.1 # via python-jose futures==3.2.0 ; python_version < "3.0" +gitdb2==2.0.5 # via gitpython +gitdb==0.6.4 +gitpython==2.1.3 google-auth==1.6.3 # via kubernetes -idna==2.8 # via requests +idna==2.8 ioflo==1.7.5 -ipaddress==1.0.22 # via cryptography, docker, kubernetes +ioloop==0.1a0 +ipaddress==1.0.22 +jaraco.functools==2.0 # via tempora jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto @@ -46,6 +54,8 @@ jsonschema==2.6.0 keyring==5.7.1 kubernetes==3.0.0 libnacl==1.6.1 +lxml==3.7.3 +mako==1.0.6 markupsafe==1.1.1 meld3==1.0.2 # via supervisor mock==2.0.0 ; python_version < "3.6" @@ -57,15 +67,20 @@ patch==1.16 pathlib2==2.3.3 # via pytest pbr==5.1.3 # via mock pluggy==0.9.0 # via pytest +portend==2.4 # via cherrypy psutil==5.6.1 py==1.8.0 # via pytest pyaml==19.4.1 # via moto pyasn1-modules==0.2.4 # via google-auth -pyasn1==0.4.5 # via pyasn1-modules, rsa -pycparser==2.19 # via cffi +pyasn1==0.4.5 +pycparser==2.19 pycryptodome==3.8.1 # via python-jose +pycryptodomex==3.8.1 +pycurl==7.43.0 pygit2==0.28.1 +pymysql==0.7.11 pyopenssl==19.0.0 +pypiwin32==223 # via cherrypy pytest-cov==2.6.1 pytest-helpers-namespace==2019.1.8 pytest-salt-runtests-bridge==2019.1.30 @@ -73,14 +88,16 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, kubernetes, moto +python-dateutil==2.8.0 python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto -pytz==2019.1 # via moto +pythonnet==2.3.0 +pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pywin32==223 pyyaml==3.13 +pyzmq==18.0.1 raet==0.6.8 requests==2.21.0 responses==0.10.6 # via moto @@ -91,16 +108,20 @@ salttesting==2017.6.1 scandir==1.10.0 # via pathlib2 sed==0.3.1 setproctitle==1.1.10 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via cryptography, docker, docker-pycreds, google-auth, kubernetes, mock, more-itertools, moto, pathlib2, pygit2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, websocket-client +singledispatch==3.4.0.3 +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, kubernetes, mock, more-itertools, moto, pathlib2, pygit2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +smmap2==2.0.5 # via gitdb2 +smmap==0.9.0 strict-rfc3339==0.7 supervisor==3.3.5 ; python_version < "3" +tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version < "3" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests virtualenv==16.4.3 websocket-client==0.40.0 # via docker, kubernetes werkzeug==0.15.2 # via moto +wheel==0.33.4 wmi==1.4.9 wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto diff --git a/requirements/static/py2.7/zeromq-windows.txt b/requirements/static/py2.7/zeromq-windows.txt index 3a42b781cbb..b630fb81dc8 100644 --- a/requirements/static/py2.7/zeromq-windows.txt +++ b/requirements/static/py2.7/zeromq-windows.txt @@ -2,41 +2,49 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/py2.7/zeromq-windows.txt -v requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/windows.in +# pip-compile -o requirements/static/py2.7/zeromq-windows.txt -v pkg/windows/req.txt pkg/windows/req_win.txt requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/windows.in # asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado -backports.ssl-match-hostname==3.7.0.1 # via docker, websocket-client +backports-abc==0.5 +backports.functools-lru-cache==1.5 # via cheroot, jaraco.functools +backports.ssl-match-hostname==3.7.0.1 backports.tempfile==1.0 # via moto backports.weakref==1.0.post1 # via backports.tempfile boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 # via kubernetes, requests, tornado -cffi==1.12.2 # via cryptography, pygit2 +certifi==2019.3.9 +cffi==1.12.2 chardet==3.0.4 # via requests +cheroot==6.5.5 # via cherrypy +cherrypy==10.2.1 colorama==0.4.1 # via pytest cookies==2.2.1 # via responses coverage==4.5.3 # via pytest-cov -cryptography==2.6.1 # via moto, pyopenssl +cryptography==2.6.1 dmidecode==0.9.0 dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==2.7.0 docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose -enum34==1.1.6 # via cryptography +enum34==1.1.6 funcsigs==1.0.2 # via mock, pytest functools32==3.2.3.post2 # via jsonschema future==0.17.1 # via python-jose futures==3.2.0 ; python_version < "3.0" +gitdb2==2.0.5 # via gitpython +gitdb==0.6.4 +gitpython==2.1.3 google-auth==1.6.3 # via kubernetes -idna==2.8 # via requests -ipaddress==1.0.22 # via cryptography, docker, kubernetes +idna==2.8 +ioloop==0.1a0 +ipaddress==1.0.22 +jaraco.functools==2.0 # via tempora jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto @@ -44,6 +52,9 @@ jsonpickle==1.1 # via aws-xray-sdk jsonschema==2.6.0 keyring==5.7.1 kubernetes==3.0.0 +libnacl==1.6.1 +lxml==3.7.3 +mako==1.0.6 markupsafe==1.1.1 meld3==1.0.2 # via supervisor mock==2.0.0 ; python_version < "3.6" @@ -55,16 +66,20 @@ patch==1.16 pathlib2==2.3.3 # via pytest pbr==5.1.3 # via mock pluggy==0.9.0 # via pytest +portend==2.4 # via cherrypy psutil==5.6.1 py==1.8.0 # via pytest pyaml==19.4.1 # via moto pyasn1-modules==0.2.4 # via google-auth -pyasn1==0.4.5 # via pyasn1-modules, rsa -pycparser==2.19 # via cffi +pyasn1==0.4.5 +pycparser==2.19 pycryptodome==3.8.1 # via python-jose pycryptodomex==3.8.1 ; sys_platform == "win32" +pycurl==7.43.0 pygit2==0.28.1 +pymysql==0.7.11 pyopenssl==19.0.0 +pypiwin32==223 # via cherrypy pytest-cov==2.6.1 pytest-helpers-namespace==2019.1.8 pytest-salt-runtests-bridge==2019.1.30 @@ -72,11 +87,12 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, kubernetes, moto +python-dateutil==2.8.0 python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto -pytz==2019.1 # via moto +pythonnet==2.3.0 +pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pywin32==223 pyyaml==3.13 @@ -90,16 +106,20 @@ salttesting==2017.6.1 scandir==1.10.0 # via pathlib2 sed==0.3.1 setproctitle==1.1.10 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via cryptography, docker, docker-pycreds, google-auth, kubernetes, mock, more-itertools, moto, pathlib2, pygit2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, websocket-client +singledispatch==3.4.0.3 +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, kubernetes, mock, more-itertools, moto, pathlib2, pygit2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client +smmap2==2.0.5 # via gitdb2 +smmap==0.9.0 strict-rfc3339==0.7 supervisor==3.3.5 ; python_version < "3" +tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version < "3" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests virtualenv==16.4.3 websocket-client==0.40.0 # via docker, kubernetes werkzeug==0.15.2 # via moto +wheel==0.33.4 wmi==1.4.9 wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto diff --git a/requirements/static/py3.5/raet-windows.txt b/requirements/static/py3.5/raet-windows.txt index e3cd5412203..56d73950d3b 100644 --- a/requirements/static/py3.5/raet-windows.txt +++ b/requirements/static/py3.5/raet-windows.txt @@ -2,23 +2,27 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/py3.5/raet-windows.txt -v requirements/base.txt requirements/raet.txt requirements/pytest.txt requirements/static/windows.in +# pip-compile -o requirements/static/py3.5/raet-windows.txt -v pkg/windows/req.txt pkg/windows/req_win.txt requirements/base.txt requirements/raet.txt requirements/pytest.txt requirements/static/windows.in # asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports.ssl-match-hostname==3.7.0.1 # via websocket-client +backports-abc==0.5 +backports.functools-lru-cache==1.5 # via cheroot +backports.ssl-match-hostname==3.7.0.1 boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 # via kubernetes, requests -cffi==1.12.2 # via cryptography, pygit2 +certifi==2019.3.9 +cffi==1.12.2 chardet==3.0.4 # via requests +cheroot==6.5.5 # via cherrypy +cherrypy==10.2.1 colorama==0.4.1 # via pytest coverage==4.5.3 # via pytest-cov -cryptography==2.6.1 # via moto, pyopenssl +cryptography==2.6.1 dmidecode==0.9.0 dnspython==1.16.0 docker-pycreds==0.4.0 # via docker @@ -26,12 +30,17 @@ docker==2.7.0 docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose # Next line explicitly commented out by pip-tools-compile because of the following regex: '^enum34==(.*)$' -# enum34==1.1.6 # via raet +# enum34==1.1.6 future==0.17.1 # via python-jose +gitdb2==2.0.5 # via gitpython +gitdb==0.6.4 +gitpython==2.1.3 google-auth==1.6.3 # via kubernetes -idna==2.8 # via requests +idna==2.8 ioflo==1.7.5 -ipaddress==1.0.22 # via kubernetes +ioloop==0.1a0 +ipaddress==1.0.22 +jaraco.functools==2.0 # via tempora jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto @@ -40,6 +49,8 @@ jsonschema==2.6.0 keyring==5.7.1 kubernetes==3.0.0 libnacl==1.6.1 +lxml==3.7.3 +mako==1.0.6 markupsafe==1.1.1 mock==2.0.0 ; python_version < "3.6" more-itertools==5.0.0 @@ -50,15 +61,20 @@ patch==1.16 pathlib2==2.3.3 # via pytest pbr==5.1.3 # via mock pluggy==0.9.0 # via pytest +portend==2.4 # via cherrypy psutil==5.6.1 py==1.8.0 # via pytest pyaml==19.4.1 # via moto pyasn1-modules==0.2.4 # via google-auth -pyasn1==0.4.5 # via pyasn1-modules, rsa -pycparser==2.19 # via cffi +pyasn1==0.4.5 +pycparser==2.19 pycryptodome==3.8.1 # via python-jose +pycryptodomex==3.8.1 +pycurl==7.43.0 pygit2==0.28.1 +pymysql==0.7.11 pyopenssl==19.0.0 +pypiwin32==223 # via cherrypy pytest-cov==2.6.1 pytest-helpers-namespace==2019.1.8 pytest-salt-runtests-bridge==2019.1.30 @@ -66,14 +82,16 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, kubernetes, moto +python-dateutil==2.8.0 python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto -pytz==2019.1 # via moto +pythonnet==2.3.0 +pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pywin32==223 pyyaml==3.13 +pyzmq==18.0.1 raet==0.6.8 requests==2.21.0 responses==0.10.6 # via moto @@ -83,14 +101,19 @@ s3transfer==0.2.0 # via boto3 salttesting==2017.6.1 sed==0.3.1 setproctitle==1.1.10 -six==1.12.0 # via cryptography, docker, docker-pycreds, google-auth, kubernetes, mock, more-itertools, moto, pathlib2, pygit2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, websocket-client +singledispatch==3.4.0.3 +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, kubernetes, mock, more-itertools, moto, pathlib2, pygit2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +smmap2==2.0.5 # via gitdb2 +smmap==0.9.0 strict-rfc3339==0.7 +tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests virtualenv==16.4.3 websocket-client==0.40.0 # via docker, kubernetes werkzeug==0.15.2 # via moto +wheel==0.33.4 wmi==1.4.9 wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto diff --git a/requirements/static/py3.5/zeromq-windows.txt b/requirements/static/py3.5/zeromq-windows.txt index 82c4231c266..79304b8165b 100644 --- a/requirements/static/py3.5/zeromq-windows.txt +++ b/requirements/static/py3.5/zeromq-windows.txt @@ -2,33 +2,43 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/py3.5/zeromq-windows.txt -v requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/windows.in +# pip-compile -o requirements/static/py3.5/zeromq-windows.txt -v pkg/windows/req.txt pkg/windows/req_win.txt requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/windows.in # asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports.ssl-match-hostname==3.7.0.1 # via websocket-client +backports-abc==0.5 +backports.functools-lru-cache==1.5 # via cheroot +backports.ssl-match-hostname==3.7.0.1 boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 # via kubernetes, requests -cffi==1.12.2 # via cryptography, pygit2 +certifi==2019.3.9 +cffi==1.12.2 chardet==3.0.4 # via requests +cheroot==6.5.5 # via cherrypy +cherrypy==10.2.1 colorama==0.4.1 # via pytest coverage==4.5.3 # via pytest-cov -cryptography==2.6.1 # via moto, pyopenssl +cryptography==2.6.1 dmidecode==0.9.0 dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==2.7.0 docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose +enum34==1.1.6 future==0.17.1 # via python-jose +gitdb2==2.0.5 # via gitpython +gitdb==0.6.4 +gitpython==2.1.3 google-auth==1.6.3 # via kubernetes -idna==2.8 # via requests -ipaddress==1.0.22 # via kubernetes +idna==2.8 +ioloop==0.1a0 +ipaddress==1.0.22 +jaraco.functools==2.0 # via tempora jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto @@ -36,6 +46,9 @@ jsonpickle==1.1 # via aws-xray-sdk jsonschema==2.6.0 keyring==5.7.1 kubernetes==3.0.0 +libnacl==1.6.1 +lxml==3.7.3 +mako==1.0.6 markupsafe==1.1.1 mock==2.0.0 ; python_version < "3.6" more-itertools==5.0.0 @@ -46,16 +59,20 @@ patch==1.16 pathlib2==2.3.3 # via pytest pbr==5.1.3 # via mock pluggy==0.9.0 # via pytest +portend==2.4 # via cherrypy psutil==5.6.1 py==1.8.0 # via pytest pyaml==19.4.1 # via moto pyasn1-modules==0.2.4 # via google-auth -pyasn1==0.4.5 # via pyasn1-modules, rsa -pycparser==2.19 # via cffi +pyasn1==0.4.5 +pycparser==2.19 pycryptodome==3.8.1 # via python-jose pycryptodomex==3.8.1 ; sys_platform == "win32" +pycurl==7.43.0 pygit2==0.28.1 +pymysql==0.7.11 pyopenssl==19.0.0 +pypiwin32==223 # via cherrypy pytest-cov==2.6.1 pytest-helpers-namespace==2019.1.8 pytest-salt-runtests-bridge==2019.1.30 @@ -63,11 +80,12 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, kubernetes, moto +python-dateutil==2.8.0 python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto -pytz==2019.1 # via moto +pythonnet==2.3.0 +pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pywin32==223 pyyaml==3.13 @@ -80,14 +98,19 @@ s3transfer==0.2.0 # via boto3 salttesting==2017.6.1 sed==0.3.1 setproctitle==1.1.10 -six==1.12.0 # via cryptography, docker, docker-pycreds, google-auth, kubernetes, mock, more-itertools, moto, pathlib2, pygit2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, websocket-client +singledispatch==3.4.0.3 +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, kubernetes, mock, more-itertools, moto, pathlib2, pygit2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client +smmap2==2.0.5 # via gitdb2 +smmap==0.9.0 strict-rfc3339==0.7 +tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests virtualenv==16.4.3 websocket-client==0.40.0 # via docker, kubernetes werkzeug==0.15.2 # via moto +wheel==0.33.4 wmi==1.4.9 wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto diff --git a/requirements/static/py3.6/raet-windows.txt b/requirements/static/py3.6/raet-windows.txt index 88be3d24e51..ce7f275ff5d 100644 --- a/requirements/static/py3.6/raet-windows.txt +++ b/requirements/static/py3.6/raet-windows.txt @@ -2,23 +2,27 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/py3.6/raet-windows.txt -v requirements/base.txt requirements/raet.txt requirements/pytest.txt requirements/static/windows.in +# pip-compile -o requirements/static/py3.6/raet-windows.txt -v pkg/windows/req.txt pkg/windows/req_win.txt requirements/base.txt requirements/raet.txt requirements/pytest.txt requirements/static/windows.in # asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports.ssl-match-hostname==3.7.0.1 # via websocket-client +backports-abc==0.5 +backports.functools-lru-cache==1.5 # via cheroot +backports.ssl-match-hostname==3.7.0.1 boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 # via kubernetes, requests -cffi==1.12.2 # via cryptography, pygit2 +certifi==2019.3.9 +cffi==1.12.2 chardet==3.0.4 # via requests +cheroot==6.5.5 # via cherrypy +cherrypy==10.2.1 colorama==0.4.1 # via pytest coverage==4.5.3 # via pytest-cov -cryptography==2.6.1 # via moto, pyopenssl +cryptography==2.6.1 dmidecode==0.9.0 dnspython==1.16.0 docker-pycreds==0.4.0 # via docker @@ -26,12 +30,17 @@ docker==2.7.0 docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose # Next line explicitly commented out by pip-tools-compile because of the following regex: '^enum34==(.*)$' -# enum34==1.1.6 # via raet +# enum34==1.1.6 future==0.17.1 # via python-jose +gitdb2==2.0.5 # via gitpython +gitdb==0.6.4 +gitpython==2.1.3 google-auth==1.6.3 # via kubernetes -idna==2.8 # via requests +idna==2.8 ioflo==1.7.5 -ipaddress==1.0.22 # via kubernetes +ioloop==0.1a0 +ipaddress==1.0.22 +jaraco.functools==2.0 # via tempora jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto @@ -40,6 +49,8 @@ jsonschema==2.6.0 keyring==5.7.1 kubernetes==3.0.0 libnacl==1.6.1 +lxml==3.7.3 +mako==1.0.6 markupsafe==1.1.1 mock==2.0.0 # via moto more-itertools==5.0.0 @@ -49,15 +60,20 @@ msgpack==0.6.1 patch==1.16 pbr==5.1.3 # via mock pluggy==0.9.0 # via pytest +portend==2.4 # via cherrypy psutil==5.6.1 py==1.8.0 # via pytest pyaml==19.4.1 # via moto pyasn1-modules==0.2.4 # via google-auth -pyasn1==0.4.5 # via pyasn1-modules, rsa -pycparser==2.19 # via cffi +pyasn1==0.4.5 +pycparser==2.19 pycryptodome==3.8.1 # via python-jose +pycryptodomex==3.8.1 +pycurl==7.43.0 pygit2==0.28.1 +pymysql==0.7.11 pyopenssl==19.0.0 +pypiwin32==223 # via cherrypy pytest-cov==2.6.1 pytest-helpers-namespace==2019.1.8 pytest-salt-runtests-bridge==2019.1.30 @@ -65,14 +81,16 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, kubernetes, moto +python-dateutil==2.8.0 python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto -pytz==2019.1 # via moto +pythonnet==2.3.0 +pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pywin32==223 pyyaml==3.13 +pyzmq==18.0.1 raet==0.6.8 requests==2.21.0 responses==0.10.6 # via moto @@ -82,14 +100,19 @@ s3transfer==0.2.0 # via boto3 salttesting==2017.6.1 sed==0.3.1 setproctitle==1.1.10 -six==1.12.0 # via cryptography, docker, docker-pycreds, google-auth, kubernetes, mock, more-itertools, moto, pygit2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, websocket-client +singledispatch==3.4.0.3 +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, kubernetes, mock, more-itertools, moto, pygit2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +smmap2==2.0.5 # via gitdb2 +smmap==0.9.0 strict-rfc3339==0.7 +tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests virtualenv==16.4.3 websocket-client==0.40.0 # via docker, kubernetes werkzeug==0.15.2 # via moto +wheel==0.33.4 wmi==1.4.9 wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto diff --git a/requirements/static/py3.6/zeromq-windows.txt b/requirements/static/py3.6/zeromq-windows.txt index fe93a34689b..6604b5f3dd0 100644 --- a/requirements/static/py3.6/zeromq-windows.txt +++ b/requirements/static/py3.6/zeromq-windows.txt @@ -2,33 +2,43 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/py3.6/zeromq-windows.txt -v requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/windows.in +# pip-compile -o requirements/static/py3.6/zeromq-windows.txt -v pkg/windows/req.txt pkg/windows/req_win.txt requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/windows.in # asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports.ssl-match-hostname==3.7.0.1 # via websocket-client +backports-abc==0.5 +backports.functools-lru-cache==1.5 # via cheroot +backports.ssl-match-hostname==3.7.0.1 boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 # via kubernetes, requests -cffi==1.12.2 # via cryptography, pygit2 +certifi==2019.3.9 +cffi==1.12.2 chardet==3.0.4 # via requests +cheroot==6.5.5 # via cherrypy +cherrypy==10.2.1 colorama==0.4.1 # via pytest coverage==4.5.3 # via pytest-cov -cryptography==2.6.1 # via moto, pyopenssl +cryptography==2.6.1 dmidecode==0.9.0 dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==2.7.0 docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose +enum34==1.1.6 future==0.17.1 # via python-jose +gitdb2==2.0.5 # via gitpython +gitdb==0.6.4 +gitpython==2.1.3 google-auth==1.6.3 # via kubernetes -idna==2.8 # via requests -ipaddress==1.0.22 # via kubernetes +idna==2.8 +ioloop==0.1a0 +ipaddress==1.0.22 +jaraco.functools==2.0 # via tempora jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto @@ -36,6 +46,9 @@ jsonpickle==1.1 # via aws-xray-sdk jsonschema==2.6.0 keyring==5.7.1 kubernetes==3.0.0 +libnacl==1.6.1 +lxml==3.7.3 +mako==1.0.6 markupsafe==1.1.1 mock==2.0.0 # via moto more-itertools==5.0.0 @@ -45,16 +58,20 @@ msgpack==0.6.1 patch==1.16 pbr==5.1.3 # via mock pluggy==0.9.0 # via pytest +portend==2.4 # via cherrypy psutil==5.6.1 py==1.8.0 # via pytest pyaml==19.4.1 # via moto pyasn1-modules==0.2.4 # via google-auth -pyasn1==0.4.5 # via pyasn1-modules, rsa -pycparser==2.19 # via cffi +pyasn1==0.4.5 +pycparser==2.19 pycryptodome==3.8.1 # via python-jose pycryptodomex==3.8.1 ; sys_platform == "win32" +pycurl==7.43.0 pygit2==0.28.1 +pymysql==0.7.11 pyopenssl==19.0.0 +pypiwin32==223 # via cherrypy pytest-cov==2.6.1 pytest-helpers-namespace==2019.1.8 pytest-salt-runtests-bridge==2019.1.30 @@ -62,11 +79,12 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, kubernetes, moto +python-dateutil==2.8.0 python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto -pytz==2019.1 # via moto +pythonnet==2.3.0 +pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pywin32==223 pyyaml==3.13 @@ -79,14 +97,19 @@ s3transfer==0.2.0 # via boto3 salttesting==2017.6.1 sed==0.3.1 setproctitle==1.1.10 -six==1.12.0 # via cryptography, docker, docker-pycreds, google-auth, kubernetes, mock, more-itertools, moto, pygit2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, websocket-client +singledispatch==3.4.0.3 +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, kubernetes, mock, more-itertools, moto, pygit2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client +smmap2==2.0.5 # via gitdb2 +smmap==0.9.0 strict-rfc3339==0.7 +tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests virtualenv==16.4.3 websocket-client==0.40.0 # via docker, kubernetes werkzeug==0.15.2 # via moto +wheel==0.33.4 wmi==1.4.9 wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto diff --git a/requirements/static/py3.7/raet-windows.txt b/requirements/static/py3.7/raet-windows.txt index a64b3b920a8..2dd76d825d7 100644 --- a/requirements/static/py3.7/raet-windows.txt +++ b/requirements/static/py3.7/raet-windows.txt @@ -2,24 +2,27 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/py3.7/raet-windows.txt -v requirements/base.txt requirements/raet.txt requirements/pytest.txt requirements/static/windows.in +# pip-compile -o requirements/static/py3.7/raet-windows.txt -v pkg/windows/req.txt pkg/windows/req_win.txt requirements/base.txt requirements/raet.txt requirements/pytest.txt requirements/static/windows.in # asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado -backports.ssl-match-hostname==3.7.0.1 # via websocket-client +backports-abc==0.5 +backports.functools-lru-cache==1.5 # via cheroot +backports.ssl-match-hostname==3.7.0.1 boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 # via kubernetes, requests, tornado -cffi==1.12.2 # via cryptography, pygit2 +certifi==2019.3.9 +cffi==1.12.2 chardet==3.0.4 # via requests +cheroot==6.5.5 # via cherrypy +cherrypy==10.2.1 colorama==0.4.1 # via pytest coverage==4.5.3 # via pytest-cov -cryptography==2.6.1 # via moto, pyopenssl +cryptography==2.6.1 dmidecode==0.9.0 dnspython==1.16.0 docker-pycreds==0.4.0 # via docker @@ -27,12 +30,17 @@ docker==2.7.0 docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose # Next line explicitly commented out by pip-tools-compile because of the following regex: '^enum34==(.*)$' -# enum34==1.1.6 # via raet +# enum34==1.1.6 future==0.17.1 # via python-jose +gitdb2==2.0.5 # via gitpython +gitdb==0.6.4 +gitpython==2.1.3 google-auth==1.6.3 # via kubernetes -idna==2.8 # via requests +idna==2.8 ioflo==1.7.5 -ipaddress==1.0.22 # via kubernetes +ioloop==0.1a0 +ipaddress==1.0.22 +jaraco.functools==2.0 # via tempora jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto @@ -41,6 +49,8 @@ jsonschema==2.6.0 keyring==5.7.1 kubernetes==3.0.0 libnacl==1.6.1 +lxml==3.7.3 +mako==1.0.6 markupsafe==1.1.1 mock==2.0.0 # via moto more-itertools==5.0.0 @@ -50,15 +60,20 @@ msgpack==0.6.1 patch==1.16 pbr==5.1.3 # via mock pluggy==0.9.0 # via pytest +portend==2.4 # via cherrypy psutil==5.6.1 py==1.8.0 # via pytest pyaml==19.4.1 # via moto pyasn1-modules==0.2.4 # via google-auth -pyasn1==0.4.5 # via pyasn1-modules, rsa -pycparser==2.19 # via cffi +pyasn1==0.4.5 +pycparser==2.19 pycryptodome==3.8.1 # via python-jose +pycryptodomex==3.8.1 +pycurl==7.43.0 pygit2==0.28.1 +pymysql==0.7.11 pyopenssl==19.0.0 +pypiwin32==223 # via cherrypy pytest-cov==2.6.1 pytest-helpers-namespace==2019.1.8 pytest-salt-runtests-bridge==2019.1.30 @@ -66,14 +81,16 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, kubernetes, moto +python-dateutil==2.8.0 python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto -pytz==2019.1 # via moto +pythonnet==2.3.0 +pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pywin32==223 pyyaml==3.13 +pyzmq==18.0.1 raet==0.6.8 requests==2.21.0 responses==0.10.6 # via moto @@ -83,15 +100,19 @@ s3transfer==0.2.0 # via boto3 salttesting==2017.6.1 sed==0.3.1 setproctitle==1.1.10 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via cryptography, docker, docker-pycreds, google-auth, kubernetes, mock, more-itertools, moto, pygit2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, websocket-client +singledispatch==3.4.0.3 +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, kubernetes, mock, more-itertools, moto, pygit2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, tempora, websocket-client +smmap2==2.0.5 # via gitdb2 +smmap==0.9.0 strict-rfc3339==0.7 +tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests virtualenv==16.4.3 websocket-client==0.40.0 # via docker, kubernetes werkzeug==0.15.2 # via moto +wheel==0.33.4 wmi==1.4.9 wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto diff --git a/requirements/static/py3.7/zeromq-windows.txt b/requirements/static/py3.7/zeromq-windows.txt index 7651b55daf1..8d64a9c0b3b 100644 --- a/requirements/static/py3.7/zeromq-windows.txt +++ b/requirements/static/py3.7/zeromq-windows.txt @@ -2,34 +2,43 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/py3.7/zeromq-windows.txt -v requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/windows.in +# pip-compile -o requirements/static/py3.7/zeromq-windows.txt -v pkg/windows/req.txt pkg/windows/req_win.txt requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/windows.in # asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado -backports.ssl-match-hostname==3.7.0.1 # via websocket-client +backports-abc==0.5 +backports.functools-lru-cache==1.5 # via cheroot +backports.ssl-match-hostname==3.7.0.1 boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 # via kubernetes, requests, tornado -cffi==1.12.2 # via cryptography, pygit2 +certifi==2019.3.9 +cffi==1.12.2 chardet==3.0.4 # via requests +cheroot==6.5.5 # via cherrypy +cherrypy==10.2.1 colorama==0.4.1 # via pytest coverage==4.5.3 # via pytest-cov -cryptography==2.6.1 # via moto, pyopenssl +cryptography==2.6.1 dmidecode==0.9.0 dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==2.7.0 docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose +enum34==1.1.6 future==0.17.1 # via python-jose +gitdb2==2.0.5 # via gitpython +gitdb==0.6.4 +gitpython==2.1.3 google-auth==1.6.3 # via kubernetes -idna==2.8 # via requests -ipaddress==1.0.22 # via kubernetes +idna==2.8 +ioloop==0.1a0 +ipaddress==1.0.22 +jaraco.functools==2.0 # via tempora jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto @@ -37,6 +46,9 @@ jsonpickle==1.1 # via aws-xray-sdk jsonschema==2.6.0 keyring==5.7.1 kubernetes==3.0.0 +libnacl==1.6.1 +lxml==3.7.3 +mako==1.0.6 markupsafe==1.1.1 mock==2.0.0 # via moto more-itertools==5.0.0 @@ -46,16 +58,20 @@ msgpack==0.6.1 patch==1.16 pbr==5.1.3 # via mock pluggy==0.9.0 # via pytest +portend==2.4 # via cherrypy psutil==5.6.1 py==1.8.0 # via pytest pyaml==19.4.1 # via moto pyasn1-modules==0.2.4 # via google-auth -pyasn1==0.4.5 # via pyasn1-modules, rsa -pycparser==2.19 # via cffi +pyasn1==0.4.5 +pycparser==2.19 pycryptodome==3.8.1 # via python-jose pycryptodomex==3.8.1 ; sys_platform == "win32" +pycurl==7.43.0 pygit2==0.28.1 +pymysql==0.7.11 pyopenssl==19.0.0 +pypiwin32==223 # via cherrypy pytest-cov==2.6.1 pytest-helpers-namespace==2019.1.8 pytest-salt-runtests-bridge==2019.1.30 @@ -63,11 +79,12 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, kubernetes, moto +python-dateutil==2.8.0 python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto -pytz==2019.1 # via moto +pythonnet==2.3.0 +pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pywin32==223 pyyaml==3.13 @@ -80,15 +97,19 @@ s3transfer==0.2.0 # via boto3 salttesting==2017.6.1 sed==0.3.1 setproctitle==1.1.10 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via cryptography, docker, docker-pycreds, google-auth, kubernetes, mock, more-itertools, moto, pygit2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, websocket-client +singledispatch==3.4.0.3 +six==1.12.0 # via cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, kubernetes, mock, more-itertools, moto, pygit2, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client +smmap2==2.0.5 # via gitdb2 +smmap==0.9.0 strict-rfc3339==0.7 +tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests virtualenv==16.4.3 websocket-client==0.40.0 # via docker, kubernetes werkzeug==0.15.2 # via moto +wheel==0.33.4 wmi==1.4.9 wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto From f60cfd8c67f156a1cdfff99320c7ee759e67347b Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 6 Jun 2019 11:44:09 +0100 Subject: [PATCH 23/31] Take into account the packaging requirements for OSX --- .pre-commit-config.yaml | 32 ++++++++++++----- pkg/osx/req.txt | 34 +++++++++--------- pkg/osx/req_ext.txt | 4 +-- requirements/static/osx.in | 1 - requirements/static/py2.7/raet-osx.txt | 44 ++++++++++++++++-------- requirements/static/py2.7/zeromq-osx.txt | 43 +++++++++++++++-------- requirements/static/py3.5/raet-osx.txt | 44 ++++++++++++++++-------- requirements/static/py3.5/zeromq-osx.txt | 42 ++++++++++++++-------- requirements/static/py3.6/raet-osx.txt | 44 ++++++++++++++++-------- requirements/static/py3.6/zeromq-osx.txt | 42 ++++++++++++++-------- requirements/static/py3.7/raet-osx.txt | 44 ++++++++++++++++-------- requirements/static/py3.7/zeromq-osx.txt | 42 ++++++++++++++-------- 12 files changed, 271 insertions(+), 145 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 703998f79b7..687fde08d88 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,12 +20,14 @@ repos: - id: pip-tools-compile alias: compile-osx-py2.7-zmq-requirements name: OSX Py2.7 ZeroMQ Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/osx\.in)$ + files: ^(pkg/osx/(req|req_ext)\.txt|requirements/((base|zeromq|pytest)\.txt|static/osx\.in))$ args: - -v - --py-version=2.7 - --platform=darwin - --out-prefix=zeromq + - --include=pkg/osx/req.txt + - --include=pkg/osx/req_ext.txt - --include=requirements/base.txt - --include=requirements/zeromq.txt - --include=requirements/pytest.txt @@ -62,12 +64,14 @@ repos: - id: pip-tools-compile alias: compile-osx-py2.7-raet-requirements name: OSX Py2.7 RAET Requirements - files: ^requirements/((base|raet|pytest)\.txt|static/osx\.in)$ + files: ^(pkg/osx/(req|req_ext)\.txt|requirements/((base|raet|pytest)\.txt|static/osx\.in))$ args: - -v - --py-version=2.7 - --out-prefix=raet - --platform=darwin + - --include=pkg/osx/req.txt + - --include=pkg/osx/req_ext.txt - --include=requirements/base.txt - --include=requirements/raet.txt - --include=requirements/pytest.txt @@ -161,12 +165,14 @@ repos: - id: pip-tools-compile alias: compile-osx-py3.5-zmq-requirements name: OSX Py3.5 ZeroMQ Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/osx\.in)$ + files: ^(pkg/osx/(req|req_ext)\.txt|requirements/((base|zeromq|pytest)\.txt|static/osx\.in))$ args: - -v - --py-version=3.5 - --platform=darwin - --out-prefix=zeromq + - --include=pkg/osx/req.txt + - --include=pkg/osx/req_ext.txt - --include=requirements/base.txt - --include=requirements/zeromq.txt - --include=requirements/pytest.txt @@ -204,12 +210,14 @@ repos: - id: pip-tools-compile alias: compile-osx-py3.5-raet-requirements name: OSX Py3.5 RAET Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/osx\.in)$ + files: ^(pkg/osx/(req|req_ext)\.txt|requirements/((base|raet|pytest)\.txt|static/osx\.in))$ args: - -v - --py-version=3.5 - --platform=darwin - --out-prefix=raet + - --include=pkg/osx/req.txt + - --include=pkg/osx/req_ext.txt - --include=requirements/base.txt - --include=requirements/raet.txt - --include=requirements/pytest.txt @@ -266,12 +274,14 @@ repos: - id: pip-tools-compile alias: compile-osx-py3.6-zmq-requirements name: OSX Py3.6 ZeroMQ Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/osx\.in)$ + files: ^(pkg/osx/(req|req_ext)\.txt|requirements/((base|zeromq|pytest)\.txt|static/osx\.in))$ args: - -v - --py-version=3.6 - --platform=darwin - --out-prefix=zeromq + - --include=pkg/osx/req.txt + - --include=pkg/osx/req_ext.txt - --include=requirements/base.txt - --include=requirements/zeromq.txt - --include=requirements/pytest.txt @@ -309,12 +319,14 @@ repos: - id: pip-tools-compile alias: compile-osx-py3.6-raet-requirements name: OSX Py3.6 RAET Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/osx\.in)$ + files: ^(pkg/osx/(req|req_ext)\.txt|requirements/((base|raet|pytest)\.txt|static/osx\.in))$ args: - -v - --py-version=3.6 - --platform=darwin - --out-prefix=raet + - --include=pkg/osx/req.txt + - --include=pkg/osx/req_ext.txt - --include=requirements/base.txt - --include=requirements/raet.txt - --include=requirements/pytest.txt @@ -371,12 +383,14 @@ repos: - id: pip-tools-compile alias: compile-osx-py3.7-zmq-requirements name: OSX Py3.7 ZeroMQ Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/osx\.in)$ + files: ^(pkg/osx/(req|req_ext)\.txt|requirements/((base|zeromq|pytest)\.txt|static/osx\.in))$ args: - -v - --py-version=3.7 - --platform=darwin - --out-prefix=zeromq + - --include=pkg/osx/req.txt + - --include=pkg/osx/req_ext.txt - --include=requirements/base.txt - --include=requirements/zeromq.txt - --include=requirements/pytest.txt @@ -414,12 +428,14 @@ repos: - id: pip-tools-compile alias: compile-osx-py3.7-raet-requirements name: OSX Py3.7 RAET Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/osx\.in)$ + files: ^(pkg/osx/(req|req_ext)\.txt|requirements/((base|raet|pytest)\.txt|static/osx\.in))$ args: - -v - --py-version=3.7 - --platform=darwin - --out-prefix=raet + - --include=pkg/osx/req.txt + - --include=pkg/osx/req_ext.txt - --include=requirements/base.txt - --include=requirements/raet.txt - --include=requirements/pytest.txt diff --git a/pkg/osx/req.txt b/pkg/osx/req.txt index 764aae3bf4f..ae5aefce634 100644 --- a/pkg/osx/req.txt +++ b/pkg/osx/req.txt @@ -2,31 +2,31 @@ apache-libcloud==2.2.1 backports.ssl_match_hostname==3.5.0.1 backports_abc==0.5 certifi -cffi==1.11.2 +cffi==1.12.2 CherryPy==13.0.0 click==6.7 enum34==1.1.6 -futures==3.1.1 +futures==3.2.0 ; python_version < "3.0" gitdb==0.6.4 -GitPython==2.1.7 -idna==2.6 -ipaddress==1.0.18 -Jinja2==2.10 +gitpython==2.1.11 +idna==2.8 +ipaddress==1.0.22 +jinja2==2.10.1 linode-python==1.1.1 Mako==1.0.7 -MarkupSafe==1.0 -msgpack-python==0.4.8 -pyasn1==0.4.2 -pycparser==2.18 -pycrypto==2.6.1 -python-dateutil==2.6.1 -python-gnupg==0.4.1 -PyYAML==3.12 -pyzmq==17.0.0 +markupsafe==1.1.1 +msgpack-python==0.5.6 +pyasn1==0.4.5 +pycparser==2.19 +pycryptodome==3.8.1 +python-dateutil==2.8.0 +python-gnupg==0.4.4 +pyyaml==3.13 +pyzmq==18.0.1 requests==2.21.0 singledispatch==3.4.0.3 -six==1.11.0 +six==1.12.0 smmap==0.9.0 timelib==0.2.4 -tornado==4.5.2 +tornado==4.5.3 vultr==1.0rc1 diff --git a/pkg/osx/req_ext.txt b/pkg/osx/req_ext.txt index df6232bab3e..76f7628a78e 100644 --- a/pkg/osx/req_ext.txt +++ b/pkg/osx/req_ext.txt @@ -1,2 +1,2 @@ -cryptography==2.1.4 -pyOpenSSL==17.5.0 +cryptography==2.6.1 +pyopenssl==19.0.0 diff --git a/requirements/static/osx.in b/requirements/static/osx.in index 1dd6d0f6804..82a9021ed9e 100644 --- a/requirements/static/osx.in +++ b/requirements/static/osx.in @@ -1,5 +1,4 @@ # This is a compilation of requirements installed on salt-jenkins git.salt state run -apache-libcloud==2.0.0 boto3 boto>=2.46.0 cffi diff --git a/requirements/static/py2.7/raet-osx.txt b/requirements/static/py2.7/raet-osx.txt index e81cf148c83..562d336ebf1 100644 --- a/requirements/static/py2.7/raet-osx.txt +++ b/requirements/static/py2.7/raet-osx.txt @@ -2,15 +2,16 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/py2.7/raet-osx.txt -v requirements/base.txt requirements/raet.txt requirements/pytest.txt requirements/static/osx.in +# pip-compile -o requirements/static/py2.7/raet-osx.txt -v pkg/osx/req.txt pkg/osx/req_ext.txt requirements/base.txt requirements/raet.txt requirements/pytest.txt requirements/static/osx.in # -apache-libcloud==2.0.0 +apache-libcloud==2.2.1 asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado -backports.ssl-match-hostname==3.7.0.1 # via docker, websocket-client +backports-abc==0.5 +backports.functools-lru-cache==1.5 # via cheroot, jaraco.functools +backports.ssl-match-hostname==3.5.0.1 backports.tempfile==1.0 # via moto backports.weakref==1.0.post1 # via backports.tempfile bcrypt==3.1.6 # via paramiko @@ -18,30 +19,36 @@ boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 # via kubernetes, requests, tornado +certifi==2019.3.9 cffi==1.12.2 chardet==3.0.4 # via requests +cheroot==6.5.5 # via cherrypy +cherrypy==13.0.0 +click==6.7 clustershell==1.8.1 cookies==2.2.1 # via responses coverage==4.5.3 # via pytest-cov croniter==0.3.29 -cryptography==2.6.1 # via moto, paramiko, pylxd, pyopenssl +cryptography==2.6.1 dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose -enum34==1.1.6 # via cryptography, raet +enum34==1.1.6 funcsigs==1.0.2 # via mock, pytest functools32==3.2.3.post2 # via jsonschema future==0.17.1 # via python-jose futures==3.2.0 ; python_version < "3.0" gitdb2==2.0.5 # via gitpython +gitdb==0.6.4 gitpython==2.1.11 google-auth==1.6.3 # via kubernetes -idna==2.8 # via requests +idna==2.8 ioflo==1.7.5 -ipaddress==1.0.22 # via cryptography, docker, kubernetes +ipaddress==1.0.22 +jaraco.classes==2.0 # via cherrypy +jaraco.functools==2.0 # via tempora jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto @@ -52,7 +59,9 @@ jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 libnacl==1.6.1 +linode-python==1.1.1 lxml==4.3.3 # via junos-eznc, ncclient +mako==1.0.7 markupsafe==1.1.1 meld3==1.0.2 # via supervisor mock==2.0.0 ; python_version < "3.6" @@ -66,12 +75,13 @@ paramiko==2.4.2 # via junos-eznc, ncclient, scp pathlib2==2.3.3 # via pytest pbr==5.1.3 # via mock, pylxd pluggy==0.9.0 # via pytest +portend==2.4 # via cherrypy psutil==5.6.1 py==1.8.0 # via pytest pyaml==19.4.1 # via moto pyasn1-modules==0.2.4 # via google-auth -pyasn1==0.4.5 # via paramiko, pyasn1-modules, rsa -pycparser==2.19 # via cffi +pyasn1==0.4.5 +pycparser==2.19 pycryptodome==3.8.1 pylxd==2.2.9 pynacl==1.3.0 # via paramiko @@ -84,13 +94,14 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto, pylxd +python-dateutil==2.8.0 python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto -pytz==2019.1 # via moto +pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 +pyzmq==18.0.1 raet==0.6.8 requests-toolbelt==0.9.1 # via pylxd requests-unixsocket==0.1.5 # via pylxd @@ -104,15 +115,18 @@ scandir==1.10.0 # via pathlib2 scp==0.13.2 # via junos-eznc selectors2==2.0.1 # via ncclient setproctitle==1.1.10 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, pathlib2, pylxd, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, websocket-client +singledispatch==3.4.0.3 +six==1.12.0 smmap2==2.0.5 # via gitdb2 +smmap==0.9.0 strict-rfc3339==0.7 supervisor==3.3.5 ; python_version < "3" +tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version < "3" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests, requests-unixsocket virtualenv==16.4.3 +vultr==1.0rc1 websocket-client==0.40.0 # via docker, kubernetes werkzeug==0.15.2 # via moto wrapt==1.11.1 # via aws-xray-sdk diff --git a/requirements/static/py2.7/zeromq-osx.txt b/requirements/static/py2.7/zeromq-osx.txt index 98f950c4ec8..7ef7297db05 100644 --- a/requirements/static/py2.7/zeromq-osx.txt +++ b/requirements/static/py2.7/zeromq-osx.txt @@ -2,15 +2,16 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/py2.7/zeromq-osx.txt -v requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/osx.in +# pip-compile -o requirements/static/py2.7/zeromq-osx.txt -v pkg/osx/req.txt pkg/osx/req_ext.txt requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/osx.in # -apache-libcloud==2.0.0 +apache-libcloud==2.2.1 asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado -backports.ssl-match-hostname==3.7.0.1 # via docker, websocket-client +backports-abc==0.5 +backports.functools-lru-cache==1.5 # via cheroot, jaraco.functools +backports.ssl-match-hostname==3.5.0.1 backports.tempfile==1.0 # via moto backports.weakref==1.0.post1 # via backports.tempfile bcrypt==3.1.6 # via paramiko @@ -18,29 +19,35 @@ boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 # via kubernetes, requests, tornado +certifi==2019.3.9 cffi==1.12.2 chardet==3.0.4 # via requests +cheroot==6.5.5 # via cherrypy +cherrypy==13.0.0 +click==6.7 clustershell==1.8.1 cookies==2.2.1 # via responses coverage==4.5.3 # via pytest-cov croniter==0.3.29 -cryptography==2.6.1 # via moto, paramiko, pylxd, pyopenssl +cryptography==2.6.1 dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose -enum34==1.1.6 # via cryptography +enum34==1.1.6 funcsigs==1.0.2 # via mock, pytest functools32==3.2.3.post2 # via jsonschema future==0.17.1 # via python-jose futures==3.2.0 ; python_version < "3.0" gitdb2==2.0.5 # via gitpython +gitdb==0.6.4 gitpython==2.1.11 google-auth==1.6.3 # via kubernetes -idna==2.8 # via requests -ipaddress==1.0.22 # via cryptography, docker, kubernetes +idna==2.8 +ipaddress==1.0.22 +jaraco.classes==2.0 # via cherrypy +jaraco.functools==2.0 # via tempora jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto @@ -50,7 +57,9 @@ junos-eznc==2.2.0 jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 +linode-python==1.1.1 lxml==4.3.3 # via junos-eznc, ncclient +mako==1.0.7 markupsafe==1.1.1 meld3==1.0.2 # via supervisor mock==2.0.0 ; python_version < "3.6" @@ -64,12 +73,13 @@ paramiko==2.4.2 # via junos-eznc, ncclient, scp pathlib2==2.3.3 # via pytest pbr==5.1.3 # via mock, pylxd pluggy==0.9.0 # via pytest +portend==2.4 # via cherrypy psutil==5.6.1 py==1.8.0 # via pytest pyaml==19.4.1 # via moto pyasn1-modules==0.2.4 # via google-auth -pyasn1==0.4.5 # via paramiko, pyasn1-modules, rsa -pycparser==2.19 # via cffi +pyasn1==0.4.5 +pycparser==2.19 # Next line explicitly commented out by pip-tools-compile because of the following regex: '^pycrypto==(.*)$' # pycrypto==2.6.1 ; sys_platform != "win32" pycryptodome==3.8.1 @@ -84,11 +94,11 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto, pylxd +python-dateutil==2.8.0 python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto -pytz==2019.1 # via moto +pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" @@ -104,15 +114,18 @@ scandir==1.10.0 # via pathlib2 scp==0.13.2 # via junos-eznc selectors2==2.0.1 # via ncclient setproctitle==1.1.10 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, pathlib2, pylxd, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, websocket-client +singledispatch==3.4.0.3 +six==1.12.0 smmap2==2.0.5 # via gitdb2 +smmap==0.9.0 strict-rfc3339==0.7 supervisor==3.3.5 ; python_version < "3" +tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version < "3" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests, requests-unixsocket virtualenv==16.4.3 +vultr==1.0rc1 websocket-client==0.40.0 # via docker, kubernetes werkzeug==0.15.2 # via moto wrapt==1.11.1 # via aws-xray-sdk diff --git a/requirements/static/py3.5/raet-osx.txt b/requirements/static/py3.5/raet-osx.txt index a84efa35b3e..96070ae0e53 100644 --- a/requirements/static/py3.5/raet-osx.txt +++ b/requirements/static/py3.5/raet-osx.txt @@ -2,41 +2,48 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/py3.5/raet-osx.txt -v requirements/base.txt requirements/raet.txt requirements/pytest.txt requirements/static/osx.in +# pip-compile -o requirements/static/py3.5/raet-osx.txt -v pkg/osx/req.txt pkg/osx/req_ext.txt requirements/base.txt requirements/raet.txt requirements/pytest.txt requirements/static/osx.in # -apache-libcloud==2.0.0 +apache-libcloud==2.2.1 asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado -backports.ssl-match-hostname==3.7.0.1 # via websocket-client +backports-abc==0.5 +backports.functools-lru-cache==1.5 # via cheroot +backports.ssl_match_hostname==3.5.0.1 bcrypt==3.1.6 # via paramiko boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 # via kubernetes, requests, tornado +certifi==2019.3.9 cffi==1.12.2 chardet==3.0.4 # via requests +cheroot==6.5.5 # via cherrypy +cherrypy==13.0.0 +click==6.7 clustershell==1.8.1 coverage==4.5.3 # via pytest-cov croniter==0.3.29 -cryptography==2.6.1 # via moto, paramiko, pylxd, pyopenssl +cryptography==2.6.1 dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose # Next line explicitly commented out by pip-tools-compile because of the following regex: '^enum34==(.*)$' -# enum34==1.1.6 # via raet +# enum34==1.1.6 future==0.17.1 # via python-jose gitdb2==2.0.5 # via gitpython +gitdb==0.6.4 gitpython==2.1.11 google-auth==1.6.3 # via kubernetes -idna==2.8 # via requests +idna==2.8 ioflo==1.7.5 -ipaddress==1.0.22 # via kubernetes +ipaddress==1.0.22 +jaraco.classes==2.0 # via cherrypy +jaraco.functools==2.0 # via tempora jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto @@ -47,7 +54,9 @@ jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 libnacl==1.6.1 +linode-python==1.1.1 lxml==4.3.3 # via junos-eznc, ncclient +mako==1.0.7 markupsafe==1.1.1 mock==2.0.0 ; python_version < "3.6" more-itertools==5.0.0 @@ -60,12 +69,13 @@ paramiko==2.4.2 # via junos-eznc, ncclient, scp pathlib2==2.3.3 # via pytest pbr==5.1.3 # via mock, pylxd pluggy==0.9.0 # via pytest +portend==2.4 # via cherrypy psutil==5.6.1 py==1.8.0 # via pytest pyaml==19.4.1 # via moto pyasn1-modules==0.2.4 # via google-auth -pyasn1==0.4.5 # via paramiko, pyasn1-modules, rsa -pycparser==2.19 # via cffi +pyasn1==0.4.5 +pycparser==2.19 pycryptodome==3.8.1 pylxd==2.2.9 pynacl==1.3.0 # via paramiko @@ -78,13 +88,14 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto, pylxd +python-dateutil==2.8.0 python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto -pytz==2019.1 # via moto +pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 +pyzmq==18.0.1 raet==0.6.8 requests-toolbelt==0.9.1 # via pylxd requests-unixsocket==0.1.5 # via pylxd @@ -96,14 +107,17 @@ s3transfer==0.2.0 # via boto3 salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, pathlib2, pylxd, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, websocket-client +singledispatch==3.4.0.3 +six==1.12.0 smmap2==2.0.5 # via gitdb2 +smmap==0.9.0 strict-rfc3339==0.7 +tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests, requests-unixsocket virtualenv==16.4.3 +vultr==1.0rc1 websocket-client==0.40.0 # via docker, kubernetes werkzeug==0.15.2 # via moto wrapt==1.11.1 # via aws-xray-sdk diff --git a/requirements/static/py3.5/zeromq-osx.txt b/requirements/static/py3.5/zeromq-osx.txt index 1b11c92b597..b676c3f23b1 100644 --- a/requirements/static/py3.5/zeromq-osx.txt +++ b/requirements/static/py3.5/zeromq-osx.txt @@ -2,38 +2,46 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/py3.5/zeromq-osx.txt -v requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/osx.in +# pip-compile -o requirements/static/py3.5/zeromq-osx.txt -v pkg/osx/req.txt pkg/osx/req_ext.txt requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/osx.in # -apache-libcloud==2.0.0 +apache-libcloud==2.2.1 asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado -backports.ssl-match-hostname==3.7.0.1 # via websocket-client +backports-abc==0.5 +backports.functools-lru-cache==1.5 # via cheroot +backports.ssl_match_hostname==3.5.0.1 bcrypt==3.1.6 # via paramiko boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 # via kubernetes, requests, tornado +certifi==2019.3.9 cffi==1.12.2 chardet==3.0.4 # via requests +cheroot==6.5.5 # via cherrypy +cherrypy==13.0.0 +click==6.7 clustershell==1.8.1 coverage==4.5.3 # via pytest-cov croniter==0.3.29 -cryptography==2.6.1 # via moto, paramiko, pylxd, pyopenssl +cryptography==2.6.1 dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose +enum34==1.1.6 future==0.17.1 # via python-jose gitdb2==2.0.5 # via gitpython +gitdb==0.6.4 gitpython==2.1.11 google-auth==1.6.3 # via kubernetes -idna==2.8 # via requests -ipaddress==1.0.22 # via kubernetes +idna==2.8 +ipaddress==1.0.22 +jaraco.classes==2.0 # via cherrypy +jaraco.functools==2.0 # via tempora jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto @@ -43,7 +51,9 @@ junos-eznc==2.2.0 jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 +linode-python==1.1.1 lxml==4.3.3 # via junos-eznc, ncclient +mako==1.0.7 markupsafe==1.1.1 mock==2.0.0 ; python_version < "3.6" more-itertools==5.0.0 @@ -56,12 +66,13 @@ paramiko==2.4.2 # via junos-eznc, ncclient, scp pathlib2==2.3.3 # via pytest pbr==5.1.3 # via mock, pylxd pluggy==0.9.0 # via pytest +portend==2.4 # via cherrypy psutil==5.6.1 py==1.8.0 # via pytest pyaml==19.4.1 # via moto pyasn1-modules==0.2.4 # via google-auth -pyasn1==0.4.5 # via paramiko, pyasn1-modules, rsa -pycparser==2.19 # via cffi +pyasn1==0.4.5 +pycparser==2.19 # Next line explicitly commented out by pip-tools-compile because of the following regex: '^pycrypto==(.*)$' # pycrypto==2.6.1 ; sys_platform != "win32" pycryptodome==3.8.1 @@ -76,11 +87,11 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto, pylxd +python-dateutil==2.8.0 python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto -pytz==2019.1 # via moto +pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" @@ -94,14 +105,17 @@ s3transfer==0.2.0 # via boto3 salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, pathlib2, pylxd, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, websocket-client +singledispatch==3.4.0.3 +six==1.12.0 smmap2==2.0.5 # via gitdb2 +smmap==0.9.0 strict-rfc3339==0.7 +tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests, requests-unixsocket virtualenv==16.4.3 +vultr==1.0rc1 websocket-client==0.40.0 # via docker, kubernetes werkzeug==0.15.2 # via moto wrapt==1.11.1 # via aws-xray-sdk diff --git a/requirements/static/py3.6/raet-osx.txt b/requirements/static/py3.6/raet-osx.txt index cbb637339f5..7583a6cce94 100644 --- a/requirements/static/py3.6/raet-osx.txt +++ b/requirements/static/py3.6/raet-osx.txt @@ -2,41 +2,48 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/py3.6/raet-osx.txt -v requirements/base.txt requirements/raet.txt requirements/pytest.txt requirements/static/osx.in +# pip-compile -o requirements/static/py3.6/raet-osx.txt -v pkg/osx/req.txt pkg/osx/req_ext.txt requirements/base.txt requirements/raet.txt requirements/pytest.txt requirements/static/osx.in # -apache-libcloud==2.0.0 +apache-libcloud==2.2.1 asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado -backports.ssl-match-hostname==3.7.0.1 # via websocket-client +backports-abc==0.5 +backports.functools-lru-cache==1.5 # via cheroot +backports.ssl_match_hostname==3.5.0.1 bcrypt==3.1.6 # via paramiko boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 # via kubernetes, requests, tornado +certifi==2019.3.9 cffi==1.12.2 chardet==3.0.4 # via requests +cheroot==6.5.5 # via cherrypy +cherrypy==13.0.0 +click==6.7 clustershell==1.8.1 coverage==4.5.3 # via pytest-cov croniter==0.3.29 -cryptography==2.6.1 # via moto, paramiko, pylxd, pyopenssl +cryptography==2.6.1 dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose # Next line explicitly commented out by pip-tools-compile because of the following regex: '^enum34==(.*)$' -# enum34==1.1.6 # via raet +# enum34==1.1.6 future==0.17.1 # via python-jose gitdb2==2.0.5 # via gitpython +gitdb==0.6.4 gitpython==2.1.11 google-auth==1.6.3 # via kubernetes -idna==2.8 # via requests +idna==2.8 ioflo==1.7.5 -ipaddress==1.0.22 # via kubernetes +ipaddress==1.0.22 +jaraco.classes==2.0 # via cherrypy +jaraco.functools==2.0 # via tempora jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto @@ -47,7 +54,9 @@ jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 libnacl==1.6.1 +linode-python==1.1.1 lxml==4.3.3 # via junos-eznc, ncclient +mako==1.0.7 markupsafe==1.1.1 mock==2.0.0 # via moto more-itertools==5.0.0 @@ -59,12 +68,13 @@ netaddr==0.7.19 # via junos-eznc paramiko==2.4.2 # via junos-eznc, ncclient, scp pbr==5.1.3 # via mock, pylxd pluggy==0.9.0 # via pytest +portend==2.4 # via cherrypy psutil==5.6.1 py==1.8.0 # via pytest pyaml==19.4.1 # via moto pyasn1-modules==0.2.4 # via google-auth -pyasn1==0.4.5 # via paramiko, pyasn1-modules, rsa -pycparser==2.19 # via cffi +pyasn1==0.4.5 +pycparser==2.19 pycryptodome==3.8.1 pylxd==2.2.9 pynacl==1.3.0 # via paramiko @@ -77,13 +87,14 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto, pylxd +python-dateutil==2.8.0 python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto -pytz==2019.1 # via moto +pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 +pyzmq==18.0.1 raet==0.6.8 requests-toolbelt==0.9.1 # via pylxd requests-unixsocket==0.1.5 # via pylxd @@ -95,14 +106,17 @@ s3transfer==0.2.0 # via boto3 salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, pylxd, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, websocket-client +singledispatch==3.4.0.3 +six==1.12.0 smmap2==2.0.5 # via gitdb2 +smmap==0.9.0 strict-rfc3339==0.7 +tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests, requests-unixsocket virtualenv==16.4.3 +vultr==1.0rc1 websocket-client==0.40.0 # via docker, kubernetes werkzeug==0.15.2 # via moto wrapt==1.11.1 # via aws-xray-sdk diff --git a/requirements/static/py3.6/zeromq-osx.txt b/requirements/static/py3.6/zeromq-osx.txt index a15178c46e3..a46994de12c 100644 --- a/requirements/static/py3.6/zeromq-osx.txt +++ b/requirements/static/py3.6/zeromq-osx.txt @@ -2,38 +2,46 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/py3.6/zeromq-osx.txt -v requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/osx.in +# pip-compile -o requirements/static/py3.6/zeromq-osx.txt -v pkg/osx/req.txt pkg/osx/req_ext.txt requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/osx.in # -apache-libcloud==2.0.0 +apache-libcloud==2.2.1 asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado -backports.ssl-match-hostname==3.7.0.1 # via websocket-client +backports-abc==0.5 +backports.functools-lru-cache==1.5 # via cheroot +backports.ssl_match_hostname==3.5.0.1 bcrypt==3.1.6 # via paramiko boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 # via kubernetes, requests, tornado +certifi==2019.3.9 cffi==1.12.2 chardet==3.0.4 # via requests +cheroot==6.5.5 # via cherrypy +cherrypy==13.0.0 +click==6.7 clustershell==1.8.1 coverage==4.5.3 # via pytest-cov croniter==0.3.29 -cryptography==2.6.1 # via moto, paramiko, pylxd, pyopenssl +cryptography==2.6.1 dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose +enum34==1.1.6 future==0.17.1 # via python-jose gitdb2==2.0.5 # via gitpython +gitdb==0.6.4 gitpython==2.1.11 google-auth==1.6.3 # via kubernetes -idna==2.8 # via requests -ipaddress==1.0.22 # via kubernetes +idna==2.8 +ipaddress==1.0.22 +jaraco.classes==2.0 # via cherrypy +jaraco.functools==2.0 # via tempora jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto @@ -43,7 +51,9 @@ junos-eznc==2.2.0 jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 +linode-python==1.1.1 lxml==4.3.3 # via junos-eznc, ncclient +mako==1.0.7 markupsafe==1.1.1 mock==2.0.0 # via moto more-itertools==5.0.0 @@ -55,12 +65,13 @@ netaddr==0.7.19 # via junos-eznc paramiko==2.4.2 # via junos-eznc, ncclient, scp pbr==5.1.3 # via mock, pylxd pluggy==0.9.0 # via pytest +portend==2.4 # via cherrypy psutil==5.6.1 py==1.8.0 # via pytest pyaml==19.4.1 # via moto pyasn1-modules==0.2.4 # via google-auth -pyasn1==0.4.5 # via paramiko, pyasn1-modules, rsa -pycparser==2.19 # via cffi +pyasn1==0.4.5 +pycparser==2.19 # Next line explicitly commented out by pip-tools-compile because of the following regex: '^pycrypto==(.*)$' # pycrypto==2.6.1 ; sys_platform != "win32" pycryptodome==3.8.1 @@ -75,11 +86,11 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto, pylxd +python-dateutil==2.8.0 python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto -pytz==2019.1 # via moto +pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" @@ -93,14 +104,17 @@ s3transfer==0.2.0 # via boto3 salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, pylxd, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, websocket-client +singledispatch==3.4.0.3 +six==1.12.0 smmap2==2.0.5 # via gitdb2 +smmap==0.9.0 strict-rfc3339==0.7 +tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests, requests-unixsocket virtualenv==16.4.3 +vultr==1.0rc1 websocket-client==0.40.0 # via docker, kubernetes werkzeug==0.15.2 # via moto wrapt==1.11.1 # via aws-xray-sdk diff --git a/requirements/static/py3.7/raet-osx.txt b/requirements/static/py3.7/raet-osx.txt index 6e14ade5707..32cee5eb2f4 100644 --- a/requirements/static/py3.7/raet-osx.txt +++ b/requirements/static/py3.7/raet-osx.txt @@ -2,41 +2,48 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/py3.7/raet-osx.txt -v requirements/base.txt requirements/raet.txt requirements/pytest.txt requirements/static/osx.in +# pip-compile -o requirements/static/py3.7/raet-osx.txt -v pkg/osx/req.txt pkg/osx/req_ext.txt requirements/base.txt requirements/raet.txt requirements/pytest.txt requirements/static/osx.in # -apache-libcloud==2.0.0 +apache-libcloud==2.2.1 asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado -backports.ssl-match-hostname==3.7.0.1 # via websocket-client +backports-abc==0.5 +backports.functools-lru-cache==1.5 # via cheroot +backports.ssl_match_hostname==3.5.0.1 bcrypt==3.1.6 # via paramiko boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 # via kubernetes, requests, tornado +certifi==2019.3.9 cffi==1.12.2 chardet==3.0.4 # via requests +cheroot==6.5.5 # via cherrypy +cherrypy==13.0.0 +click==6.7 clustershell==1.8.1 coverage==4.5.3 # via pytest-cov croniter==0.3.29 -cryptography==2.6.1 # via moto, paramiko, pylxd, pyopenssl +cryptography==2.6.1 dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose # Next line explicitly commented out by pip-tools-compile because of the following regex: '^enum34==(.*)$' -# enum34==1.1.6 # via raet +# enum34==1.1.6 future==0.17.1 # via python-jose gitdb2==2.0.5 # via gitpython +gitdb==0.6.4 gitpython==2.1.11 google-auth==1.6.3 # via kubernetes -idna==2.8 # via requests +idna==2.8 ioflo==1.7.5 -ipaddress==1.0.22 # via kubernetes +ipaddress==1.0.22 +jaraco.classes==2.0 # via cherrypy +jaraco.functools==2.0 # via tempora jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto @@ -47,7 +54,9 @@ jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 libnacl==1.6.1 +linode-python==1.1.1 lxml==4.3.3 # via junos-eznc, ncclient +mako==1.0.7 markupsafe==1.1.1 mock==2.0.0 # via moto more-itertools==5.0.0 @@ -59,12 +68,13 @@ netaddr==0.7.19 # via junos-eznc paramiko==2.4.2 # via junos-eznc, ncclient, scp pbr==5.1.3 # via mock, pylxd pluggy==0.9.0 # via pytest +portend==2.4 # via cherrypy psutil==5.6.1 py==1.8.0 # via pytest pyaml==19.4.1 # via moto pyasn1-modules==0.2.4 # via google-auth -pyasn1==0.4.5 # via paramiko, pyasn1-modules, rsa -pycparser==2.19 # via cffi +pyasn1==0.4.5 +pycparser==2.19 pycryptodome==3.8.1 pylxd==2.2.9 pynacl==1.3.0 # via paramiko @@ -77,13 +87,14 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto, pylxd +python-dateutil==2.8.0 python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto -pytz==2019.1 # via moto +pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 +pyzmq==18.0.1 raet==0.6.8 requests-toolbelt==0.9.1 # via pylxd requests-unixsocket==0.1.5 # via pylxd @@ -95,14 +106,17 @@ s3transfer==0.2.0 # via boto3 salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, pylxd, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, raet, responses, salttesting, singledispatch, websocket-client +singledispatch==3.4.0.3 +six==1.12.0 smmap2==2.0.5 # via gitdb2 +smmap==0.9.0 strict-rfc3339==0.7 +tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests, requests-unixsocket virtualenv==16.4.3 +vultr==1.0rc1 websocket-client==0.40.0 # via docker, kubernetes werkzeug==0.15.2 # via moto wrapt==1.11.1 # via aws-xray-sdk diff --git a/requirements/static/py3.7/zeromq-osx.txt b/requirements/static/py3.7/zeromq-osx.txt index 26ef52d1a70..4729cfb9c8b 100644 --- a/requirements/static/py3.7/zeromq-osx.txt +++ b/requirements/static/py3.7/zeromq-osx.txt @@ -2,38 +2,46 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -o requirements/static/py3.7/zeromq-osx.txt -v requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/osx.in +# pip-compile -o requirements/static/py3.7/zeromq-osx.txt -v pkg/osx/req.txt pkg/osx/req_ext.txt requirements/base.txt requirements/zeromq.txt requirements/pytest.txt requirements/static/osx.in # -apache-libcloud==2.0.0 +apache-libcloud==2.2.1 asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado -backports.ssl-match-hostname==3.7.0.1 # via websocket-client +backports-abc==0.5 +backports.functools-lru-cache==1.5 # via cheroot +backports.ssl_match_hostname==3.5.0.1 bcrypt==3.1.6 # via paramiko boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 # via kubernetes, requests, tornado +certifi==2019.3.9 cffi==1.12.2 chardet==3.0.4 # via requests +cheroot==6.5.5 # via cherrypy +cherrypy==13.0.0 +click==6.7 clustershell==1.8.1 coverage==4.5.3 # via pytest-cov croniter==0.3.29 -cryptography==2.6.1 # via moto, paramiko, pylxd, pyopenssl +cryptography==2.6.1 dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose +enum34==1.1.6 future==0.17.1 # via python-jose gitdb2==2.0.5 # via gitpython +gitdb==0.6.4 gitpython==2.1.11 google-auth==1.6.3 # via kubernetes -idna==2.8 # via requests -ipaddress==1.0.22 # via kubernetes +idna==2.8 +ipaddress==1.0.22 +jaraco.classes==2.0 # via cherrypy +jaraco.functools==2.0 # via tempora jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto @@ -43,7 +51,9 @@ junos-eznc==2.2.0 jxmlease==1.0.1 keyring==5.7.1 kubernetes==3.0.0 +linode-python==1.1.1 lxml==4.3.3 # via junos-eznc, ncclient +mako==1.0.7 markupsafe==1.1.1 mock==2.0.0 # via moto more-itertools==5.0.0 @@ -55,12 +65,13 @@ netaddr==0.7.19 # via junos-eznc paramiko==2.4.2 # via junos-eznc, ncclient, scp pbr==5.1.3 # via mock, pylxd pluggy==0.9.0 # via pytest +portend==2.4 # via cherrypy psutil==5.6.1 py==1.8.0 # via pytest pyaml==19.4.1 # via moto pyasn1-modules==0.2.4 # via google-auth -pyasn1==0.4.5 # via paramiko, pyasn1-modules, rsa -pycparser==2.19 # via cffi +pyasn1==0.4.5 +pycparser==2.19 # Next line explicitly commented out by pip-tools-compile because of the following regex: '^pycrypto==(.*)$' # pycrypto==2.6.1 ; sys_platform != "win32" pycryptodome==3.8.1 @@ -75,11 +86,11 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto, pylxd +python-dateutil==2.8.0 python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto -pytz==2019.1 # via moto +pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==3.13 pyzmq==18.0.1 ; python_version != "3.4" @@ -93,14 +104,17 @@ s3transfer==0.2.0 # via boto3 salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, pylxd, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, websocket-client +singledispatch==3.4.0.3 +six==1.12.0 smmap2==2.0.5 # via gitdb2 +smmap==0.9.0 strict-rfc3339==0.7 +tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests, requests-unixsocket virtualenv==16.4.3 +vultr==1.0rc1 websocket-client==0.40.0 # via docker, kubernetes werkzeug==0.15.2 # via moto wrapt==1.11.1 # via aws-xray-sdk From 3a7b18d9494f8aad03f45084d336eab8a57eb218 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 7 Jun 2019 21:09:02 +0100 Subject: [PATCH 24/31] Fix `unit.templates.test_jinja.TestCustomExtensions.test_http_query` --- salt/utils/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/http.py b/salt/utils/http.py index b187c566b50..2976e447456 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -464,7 +464,7 @@ def query(url, not isinstance(result_text, six.text_type): result_text = result_text.decode(res_params['charset']) if six.PY3 and isinstance(result_text, bytes): - result_text = result.body.decode('utf-8') + result_text = result_text.decode('utf-8') ret['body'] = result_text else: # Tornado From 4dd6298ef1307047aae073184907cb72b431a8da Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 11 Jun 2019 14:25:58 +0100 Subject: [PATCH 25/31] Reflect the lint requirements update on the static lint requirements --- requirements/static/lint.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/static/lint.in b/requirements/static/lint.in index a3e9e972e1b..d0206b8f777 100644 --- a/requirements/static/lint.in +++ b/requirements/static/lint.in @@ -1,3 +1,3 @@ # Lint requirements pylint==1.6.5 -SaltPyLint>=v2017.3.6 +SaltPyLint>=v2019.6.7 From bada5a184ec5b125b5ee0428166e1b989f090b8b Mon Sep 17 00:00:00 2001 From: Jim McStanton Date: Fri, 14 Jun 2019 09:24:18 -0500 Subject: [PATCH 26/31] Cleared up sentence in unit test docs --- doc/topics/development/tests/unit.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/topics/development/tests/unit.rst b/doc/topics/development/tests/unit.rst index d129ad478ef..fa9c2d8657f 100644 --- a/doc/topics/development/tests/unit.rst +++ b/doc/topics/development/tests/unit.rst @@ -99,8 +99,8 @@ Salt loader modules use a series of globally available dunder variables, ``__salt__``, ``__opts__``, ``__pillar__``, etc. To facilitate testing these modules a mixin class was created, ``LoaderModuleMockMixin`` which can be found in ``tests/support/mixins.py``. The reason for the existence of this class is -because historiclly and because it was easier, one would add these dunder -variables directly on the imported module. This however, introduces unexpected +because historically one would add these dunder +variables directly on the imported module. This, however, introduces unexpected behavior when running the full test suite since those attributes would not be removed once we were done testing the module and would therefore leak to other modules being tested with unpredictable results. This is the kind of work that From 9729e9ec433bb4d3f5b60143d151eca9d7b2b3e7 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 14 Jun 2019 13:07:49 +0100 Subject: [PATCH 27/31] Workaround nox's install only flag To get some information from the system, which we then use to choose the appropriate static requirements file, we need to run some commands, something that nox will refuse to do if `--install-only` is passed. We work around it by manually patching the value of `session._runner.global_config.install_only` for the commands that we MUST run, and only those and then we set it back to the value it had before. For additional information about why we have to do this, please see: https://github.com/theacodes/nox/pull/181 --- noxfile.py | 66 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/noxfile.py b/noxfile.py index 27deca25813..0b99eb5a6aa 100644 --- a/noxfile.py +++ b/noxfile.py @@ -55,15 +55,22 @@ def _get_session_python_version_info(session): try: version_info = session._runner._real_python_version_info except AttributeError: - session_py_version = session.run( - 'python', '-c' - 'import sys; sys.stdout.write("{}.{}.{}".format(*sys.version_info))', - silent=True, - log=False, - bypass_install_only=True - ) - version_info = tuple(int(part) for part in session_py_version.split('.') if part.isdigit()) - session._runner._real_python_version_info = version_info + old_install_only_value = session._runner.global_config.install_only + try: + # Force install only to be false for the following chunk of code + # For additional information as to why see: + # https://github.com/theacodes/nox/pull/181 + session._runner.global_config.install_only = False + session_py_version = session.run( + 'python', '-c' + 'import sys; sys.stdout.write("{}.{}.{}".format(*sys.version_info))', + silent=True, + log=False, + ) + version_info = tuple(int(part) for part in session_py_version.split('.') if part.isdigit()) + session._runner._real_python_version_info = version_info + finally: + session._runner.global_config.install_only = old_install_only_value return version_info @@ -71,14 +78,21 @@ def _get_session_python_site_packages_dir(session): try: site_packages_dir = session._runner._site_packages_dir except AttributeError: - site_packages_dir = session.run( - 'python', '-c' - 'import sys; from distutils.sysconfig import get_python_lib; sys.stdout.write(get_python_lib())', - silent=True, - log=False, - bypass_install_only=True - ) - session._runner._site_packages_dir = site_packages_dir + old_install_only_value = session._runner.global_config.install_only + try: + # Force install only to be false for the following chunk of code + # For additional information as to why see: + # https://github.com/theacodes/nox/pull/181 + session._runner.global_config.install_only = False + site_packages_dir = session.run( + 'python', '-c' + 'import sys; from distutils.sysconfig import get_python_lib; sys.stdout.write(get_python_lib())', + silent=True, + log=False, + ) + session._runner._site_packages_dir = site_packages_dir + finally: + session._runner.global_config.install_only = old_install_only_value return site_packages_dir @@ -94,11 +108,19 @@ def _get_distro_info(session): distro = session._runner._distro except AttributeError: # The distro package doesn't output anything for Windows - session.install('--progress-bar=off', 'distro', silent=PIP_INSTALL_SILENT) - output = session.run('distro', '-j', silent=True, bypass_install_only=True) - distro = json.loads(output.strip()) - session.log('Distro information:\n%s', pprint.pformat(distro)) - session._runner._distro = distro + old_install_only_value = session._runner.global_config.install_only + try: + # Force install only to be false for the following chunk of code + # For additional information as to why see: + # https://github.com/theacodes/nox/pull/181 + session._runner.global_config.install_only = False + session.install('--progress-bar=off', 'distro', silent=PIP_INSTALL_SILENT) + output = session.run('distro', '-j', silent=True) + distro = json.loads(output.strip()) + session.log('Distro information:\n%s', pprint.pformat(distro)) + session._runner._distro = distro + finally: + session._runner.global_config.install_only = old_install_only_value return distro From 0363cf6542dbed790e496ca6d5086ca970ef47ae Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 7 Jun 2019 21:09:02 +0100 Subject: [PATCH 28/31] Fix `unit.templates.test_jinja.TestCustomExtensions.test_http_query` --- salt/utils/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/http.py b/salt/utils/http.py index b187c566b50..2976e447456 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -464,7 +464,7 @@ def query(url, not isinstance(result_text, six.text_type): result_text = result_text.decode(res_params['charset']) if six.PY3 and isinstance(result_text, bytes): - result_text = result.body.decode('utf-8') + result_text = result_text.decode('utf-8') ret['body'] = result_text else: # Tornado From c661e8e4940a80d2470c90c66625d5a68e8431dc Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 18 Jun 2019 23:35:52 +0100 Subject: [PATCH 29/31] Update pipelines to work on old and new jenkins --- .ci/docs | 154 +++++++++------- .ci/kitchen-centos6-py2 | 188 ++++++++++--------- .ci/kitchen-centos7-py2 | 188 ++++++++++--------- .ci/kitchen-centos7-py3 | 188 ++++++++++--------- .ci/kitchen-debian8-py2 | 188 ++++++++++--------- .ci/kitchen-debian8-py3 | 188 ++++++++++--------- .ci/kitchen-debian9-py2 | 188 ++++++++++--------- .ci/kitchen-debian9-py3 | 188 ++++++++++--------- .ci/kitchen-fedora29-py2 | 188 ++++++++++--------- .ci/kitchen-fedora29-py3 | 188 ++++++++++--------- .ci/kitchen-ubuntu1604-py2 | 190 +++++++++++--------- .ci/kitchen-ubuntu1604-py3 | 190 +++++++++++--------- .ci/kitchen-ubuntu1804-py2 | 188 ++++++++++--------- .ci/kitchen-ubuntu1804-py3 | 188 ++++++++++--------- .ci/kitchen-windows2016-py2 | 189 ++++++++++--------- .ci/kitchen-windows2016-py3 | 188 ++++++++++--------- .ci/lint | 349 +++++++++++++++++++----------------- 17 files changed, 1880 insertions(+), 1448 deletions(-) diff --git a/.ci/docs b/.ci/docs index c76fc4ef8bf..9a85c6a770f 100644 --- a/.ci/docs +++ b/.ci/docs @@ -1,70 +1,92 @@ -pipeline { - agent { - label 'docs' - } - options { - timestamps() - ansiColor('xterm') - timeout(time: 2, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '10')) - } - environment { - PYENV_ROOT = "/usr/local/pyenv" - PATH = "$PYENV_ROOT/bin:$PATH" - PY_COLORS = 1 - SPHINXOPTS = "-W" - } - stages { - stage('github-pending') { - steps { - githubNotify credentialsId: 'test-jenkins-credentials', - description: 'Testing docs...', - status: 'PENDING', - context: "jenkins/pr/docs" +// Define the maximum time, in hours, that a test run should run for +def global_timeout = 2 + +properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), + [ + $class: 'ScannerJobProperty', doNotScan: false + ], + [ + $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false + ], +]) + +def shell_header + +timeout(time: global_timeout, unit: 'HOURS') { + node('docs') { + ansiColor('xterm') { + timestamps { + try { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: 'Testing docs...', + status: 'PENDING', + context: "jenkins/pr/docs" + } + shell_header = 'export PYENV_ROOT="/usr/local/pyenv"\nexport PATH="$PYENV_ROOT/bin:$PATH"' + } else { + shell_header = '' + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + + // Setup the kitchen required bundle + stage('Setup') { + sh shell_header + ''' + eval "$(pyenv init -)" + pyenv --version + pyenv install --skip-existing 3.6.8 + pyenv shell 3.6.8 + python --version + pip install -U https://github.com/s0undt3ch/nox/archive/hotfix/py2.zip#egg=Nox==2018.10.17 + nox --version + ''' + } + + stage('Build') { + sh shell_header + ''' + eval "$(pyenv init -)" + pyenv shell 3.6.8 + nox -e docs + ''' + archiveArtifacts artifacts: 'doc/doc-archive.tar.gz' + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' + } finally { + cleanWs notFailBuild: true + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: 'The docs job has passed', + status: 'SUCCESS', + context: "jenkins/pr/docs" + } + } else { + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: 'The docs job has failed', + status: 'FAILURE', + context: "jenkins/pr/docs" + } + try { + slackSend channel: "#jenkins-prod-pr", + color: '#FF0000', + message: "FAILED: PR-Job: '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" + } catch (Exception e) { + sh 'echo Failed to send the Slack notification' + } + } + } } } - stage('setup') { - steps { - sh ''' - eval "$(pyenv init -)" - pyenv --version - pyenv install --skip-existing 3.6.8 - pyenv shell 3.6.8 - python --version - pip install -U https://github.com/s0undt3ch/nox/archive/hotfix/py2.zip#egg=Nox==2018.10.17 - nox --version - ''' - } - } - stage('build') { - steps { - sh ''' - eval "$(pyenv init -)" - pyenv shell 3.6.8 - nox -e docs - ''' - archiveArtifacts artifacts: 'doc/doc-archive.tar.gz' - } - } - } - post { - always { - cleanWs() - } - success { - githubNotify credentialsId: 'test-jenkins-credentials', - description: 'The docs job has passed', - status: 'SUCCESS', - context: "jenkins/pr/docs" - } - failure { - githubNotify credentialsId: 'test-jenkins-credentials', - description: 'The docs job has failed', - status: 'FAILURE', - context: "jenkins/pr/docs" - slackSend channel: "#jenkins-prod-pr", - color: '#FF0000', - message: "FAILED: PR-Job: '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" - } } } + +// vim: ft=groovy diff --git a/.ci/kitchen-centos6-py2 b/.ci/kitchen-centos6-py2 index 13fb59af362..e0212799ca3 100644 --- a/.ci/kitchen-centos6-py2 +++ b/.ci/kitchen-centos6-py2 @@ -4,6 +4,11 @@ def testrun_timeout = 6 // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; +def distro_name = 'centos' +def distro_version = '6' +def python_version = 'py2' +def golden_images_branch = '2018.3' + properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ @@ -16,101 +21,122 @@ properties([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) + timeout(time: global_timeout, unit: 'HOURS') { node('kitchen-slave') { - timestamps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + credentialsId: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', - 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'NOX_ENV_NAME=runtests-zeromq', - 'NOX_PASSTHROUGH_OPTS=--ssh-tests', - 'NOX_ENABLE_FROM_FILENAMES=true', - 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=centos6,py2', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py2', - 'TEST_PLATFORM=centos-6', - 'PY_COLORS=1', - "FORCE_FULL=${params.runFull}", - ]) { - // Set the GH status even before cloning the repo - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - // Checkout the repo - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { + timestamps { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', + 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'NOX_ENV_NAME=runtests-zeromq', + 'NOX_ENABLE_FROM_FILENAMES=true', + 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", + "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", + 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + "TEST_SUITE=${python_version}", + "TEST_PLATFORM=${distro_name}-${distro_version}", + "FORCE_FULL=${params.runFull}", + ]) { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + // Setup the kitchen required bundle stage('setup-bundle') { sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } - try { - stage('run kitchen') { - timeout(time: testrun_timeout, unit: 'HOURS') { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' - } - } - } - } - } finally { - stage('cleanup kitchen') { - script { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - } - } - archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' - } - stage('report code coverage') { - script { - withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { - sh ''' - if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then - curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" - fi - ''' - } - } + + stage('Create VM') { + retry(3) { + sh ''' + t=$(shuf -i 1-15 -n 1); echo "Sleeping $t seconds"; sleep $t + bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' } } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { try { - junit 'artifacts/xml-unittests-output/*.xml' + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem || ssh-add ~/.ssh/kitchen.pem' + try { + timeout(time: testrun_timeout, unit: 'HOURS') { + stage('Converge VM') { + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + } + stage('Run Tests') { + withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) { + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } + } + } finally { + try { + stage('Download Artefacts') { + withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){ + sh ''' + bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0 + ''' + } + } + archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + stage('Cleanup') { + sh ''' + bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' + } + stage('Upload Coverage') { + script { + withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { + sh ''' + if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then + curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" + fi + ''' + } + } + } + } + } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' } finally { cleanWs notFailBuild: true if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } try { slackSend channel: "#jenkins-prod-pr", color: '#FF0000', @@ -127,4 +153,4 @@ timeout(time: global_timeout, unit: 'HOURS') { } } -// vi: ft=groovy +// vim: ft=groovy diff --git a/.ci/kitchen-centos7-py2 b/.ci/kitchen-centos7-py2 index 88ac5de726f..809d91db22e 100644 --- a/.ci/kitchen-centos7-py2 +++ b/.ci/kitchen-centos7-py2 @@ -4,6 +4,11 @@ def testrun_timeout = 6 // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; +def distro_name = 'centos' +def distro_version = '7' +def python_version = 'py2' +def golden_images_branch = '2018.3' + properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ @@ -16,101 +21,122 @@ properties([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) + timeout(time: global_timeout, unit: 'HOURS') { node('kitchen-slave') { - timestamps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + credentialsId: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', - 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'NOX_ENV_NAME=runtests-zeromq', - 'NOX_PASSTHROUGH_OPTS=--ssh-tests', - 'NOX_ENABLE_FROM_FILENAMES=true', - 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=centos7,py2', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py2', - 'TEST_PLATFORM=centos-7', - 'PY_COLORS=1', - "FORCE_FULL=${params.runFull}", - ]) { - // Set the GH status even before cloning the repo - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - // Checkout the repo - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { + timestamps { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', + 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'NOX_ENV_NAME=runtests-zeromq', + 'NOX_ENABLE_FROM_FILENAMES=true', + 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", + "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", + 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + "TEST_SUITE=${python_version}", + "TEST_PLATFORM=${distro_name}-${distro_version}", + "FORCE_FULL=${params.runFull}", + ]) { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + // Setup the kitchen required bundle stage('setup-bundle') { sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } - try { - stage('run kitchen') { - timeout(time: testrun_timeout, unit: 'HOURS') { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' - } - } - } - } - } finally { - stage('cleanup kitchen') { - script { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - } - } - archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' - } - stage('report code coverage') { - script { - withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { - sh ''' - if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then - curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" - fi - ''' - } - } + + stage('Create VM') { + retry(3) { + sh ''' + t=$(shuf -i 1-15 -n 1); echo "Sleeping $t seconds"; sleep $t + bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' } } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { try { - junit 'artifacts/xml-unittests-output/*.xml' + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem || ssh-add ~/.ssh/kitchen.pem' + try { + timeout(time: testrun_timeout, unit: 'HOURS') { + stage('Converge VM') { + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + } + stage('Run Tests') { + withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) { + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } + } + } finally { + try { + stage('Download Artefacts') { + withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){ + sh ''' + bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0 + ''' + } + } + archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + stage('Cleanup') { + sh ''' + bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' + } + stage('Upload Coverage') { + script { + withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { + sh ''' + if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then + curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" + fi + ''' + } + } + } + } + } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' } finally { cleanWs notFailBuild: true if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } try { slackSend channel: "#jenkins-prod-pr", color: '#FF0000', @@ -127,4 +153,4 @@ timeout(time: global_timeout, unit: 'HOURS') { } } -// vi: ft=groovy +// vim: ft=groovy diff --git a/.ci/kitchen-centos7-py3 b/.ci/kitchen-centos7-py3 index 80c82fdb5e0..b6895823547 100644 --- a/.ci/kitchen-centos7-py3 +++ b/.ci/kitchen-centos7-py3 @@ -4,6 +4,11 @@ def testrun_timeout = 6 // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; +def distro_name = 'centos' +def distro_version = '7' +def python_version = 'py3' +def golden_images_branch = '2018.3' + properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ @@ -16,101 +21,122 @@ properties([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) + timeout(time: global_timeout, unit: 'HOURS') { node('kitchen-slave') { - timestamps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + credentialsId: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', - 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'NOX_ENV_NAME=runtests-zeromq', - 'NOX_PASSTHROUGH_OPTS=--ssh-tests', - 'NOX_ENABLE_FROM_FILENAMES=true', - 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=centos7,py3', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py3', - 'TEST_PLATFORM=centos-7', - 'PY_COLORS=1', - "FORCE_FULL=${params.runFull}", - ]) { - // Set the GH status even before cloning the repo - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - // Checkout the repo - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { + timestamps { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', + 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'NOX_ENV_NAME=runtests-zeromq', + 'NOX_ENABLE_FROM_FILENAMES=true', + 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", + "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", + 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + "TEST_SUITE=${python_version}", + "TEST_PLATFORM=${distro_name}-${distro_version}", + "FORCE_FULL=${params.runFull}", + ]) { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + // Setup the kitchen required bundle stage('setup-bundle') { sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } - try { - stage('run kitchen') { - timeout(time: testrun_timeout, unit: 'HOURS') { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' - } - } - } - } - } finally { - stage('cleanup kitchen') { - script { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - } - } - archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' - } - stage('report code coverage') { - script { - withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { - sh ''' - if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then - curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" - fi - ''' - } - } + + stage('Create VM') { + retry(3) { + sh ''' + t=$(shuf -i 1-15 -n 1); echo "Sleeping $t seconds"; sleep $t + bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' } } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { try { - junit 'artifacts/xml-unittests-output/*.xml' + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem || ssh-add ~/.ssh/kitchen.pem' + try { + timeout(time: testrun_timeout, unit: 'HOURS') { + stage('Converge VM') { + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + } + stage('Run Tests') { + withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) { + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } + } + } finally { + try { + stage('Download Artefacts') { + withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){ + sh ''' + bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0 + ''' + } + } + archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + stage('Cleanup') { + sh ''' + bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' + } + stage('Upload Coverage') { + script { + withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { + sh ''' + if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then + curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" + fi + ''' + } + } + } + } + } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' } finally { cleanWs notFailBuild: true if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } try { slackSend channel: "#jenkins-prod-pr", color: '#FF0000', @@ -127,4 +153,4 @@ timeout(time: global_timeout, unit: 'HOURS') { } } -// vi: ft=groovy +// vim: ft=groovy diff --git a/.ci/kitchen-debian8-py2 b/.ci/kitchen-debian8-py2 index 13b1e1ae6b2..83418d261fb 100644 --- a/.ci/kitchen-debian8-py2 +++ b/.ci/kitchen-debian8-py2 @@ -4,6 +4,11 @@ def testrun_timeout = 6 // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; +def distro_name = 'debian' +def distro_version = '8' +def python_version = 'py2' +def golden_images_branch = '2018.3' + properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ @@ -16,101 +21,122 @@ properties([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) + timeout(time: global_timeout, unit: 'HOURS') { node('kitchen-slave') { - timestamps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + credentialsId: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', - 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'NOX_ENV_NAME=runtests-zeromq', - 'NOX_PASSTHROUGH_OPTS=--ssh-tests', - 'NOX_ENABLE_FROM_FILENAMES=true', - 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=debian8,py2', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py2', - 'TEST_PLATFORM=debian-8', - 'PY_COLORS=1', - "FORCE_FULL=${params.runFull}", - ]) { - // Set the GH status even before cloning the repo - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - // Checkout the repo - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { + timestamps { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', + 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'NOX_ENV_NAME=runtests-zeromq', + 'NOX_ENABLE_FROM_FILENAMES=true', + 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", + "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", + 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + "TEST_SUITE=${python_version}", + "TEST_PLATFORM=${distro_name}-${distro_version}", + "FORCE_FULL=${params.runFull}", + ]) { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + // Setup the kitchen required bundle stage('setup-bundle') { sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } - try { - stage('run kitchen') { - timeout(time: testrun_timeout, unit: 'HOURS') { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' - } - } - } - } - } finally { - stage('cleanup kitchen') { - script { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - } - } - archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' - } - stage('report code coverage') { - script { - withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { - sh ''' - if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then - curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" - fi - ''' - } - } + + stage('Create VM') { + retry(3) { + sh ''' + t=$(shuf -i 1-15 -n 1); echo "Sleeping $t seconds"; sleep $t + bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' } } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { try { - junit 'artifacts/xml-unittests-output/*.xml' + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem || ssh-add ~/.ssh/kitchen.pem' + try { + timeout(time: testrun_timeout, unit: 'HOURS') { + stage('Converge VM') { + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + } + stage('Run Tests') { + withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) { + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } + } + } finally { + try { + stage('Download Artefacts') { + withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){ + sh ''' + bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0 + ''' + } + } + archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + stage('Cleanup') { + sh ''' + bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' + } + stage('Upload Coverage') { + script { + withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { + sh ''' + if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then + curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" + fi + ''' + } + } + } + } + } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' } finally { cleanWs notFailBuild: true if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } try { slackSend channel: "#jenkins-prod-pr", color: '#FF0000', @@ -127,4 +153,4 @@ timeout(time: global_timeout, unit: 'HOURS') { } } -// vi: ft=groovy +// vim: ft=groovy diff --git a/.ci/kitchen-debian8-py3 b/.ci/kitchen-debian8-py3 index 4999103ac41..371fdfdf6c8 100644 --- a/.ci/kitchen-debian8-py3 +++ b/.ci/kitchen-debian8-py3 @@ -4,6 +4,11 @@ def testrun_timeout = 6 // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; +def distro_name = 'debian' +def distro_version = '8' +def python_version = 'py3' +def golden_images_branch = '2018.3' + properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ @@ -16,101 +21,122 @@ properties([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) + timeout(time: global_timeout, unit: 'HOURS') { node('kitchen-slave') { - timestamps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + credentialsId: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', - 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'NOX_ENV_NAME=runtests-zeromq', - 'NOX_PASSTHROUGH_OPTS=--ssh-tests', - 'NOX_ENABLE_FROM_FILENAMES=true', - 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=debian8,py3', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py3', - 'TEST_PLATFORM=debian-8', - 'PY_COLORS=1', - "FORCE_FULL=${params.runFull}", - ]) { - // Set the GH status even before cloning the repo - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - // Checkout the repo - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { + timestamps { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', + 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'NOX_ENV_NAME=runtests-zeromq', + 'NOX_ENABLE_FROM_FILENAMES=true', + 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", + "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", + 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + "TEST_SUITE=${python_version}", + "TEST_PLATFORM=${distro_name}-${distro_version}", + "FORCE_FULL=${params.runFull}", + ]) { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + // Setup the kitchen required bundle stage('setup-bundle') { sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } - try { - stage('run kitchen') { - timeout(time: testrun_timeout, unit: 'HOURS') { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' - } - } - } - } - } finally { - stage('cleanup kitchen') { - script { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - } - } - archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' - } - stage('report code coverage') { - script { - withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { - sh ''' - if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then - curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" - fi - ''' - } - } + + stage('Create VM') { + retry(3) { + sh ''' + t=$(shuf -i 1-15 -n 1); echo "Sleeping $t seconds"; sleep $t + bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' } } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { try { - junit 'artifacts/xml-unittests-output/*.xml' + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem || ssh-add ~/.ssh/kitchen.pem' + try { + timeout(time: testrun_timeout, unit: 'HOURS') { + stage('Converge VM') { + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + } + stage('Run Tests') { + withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) { + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } + } + } finally { + try { + stage('Download Artefacts') { + withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){ + sh ''' + bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0 + ''' + } + } + archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + stage('Cleanup') { + sh ''' + bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' + } + stage('Upload Coverage') { + script { + withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { + sh ''' + if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then + curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" + fi + ''' + } + } + } + } + } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' } finally { cleanWs notFailBuild: true if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } try { slackSend channel: "#jenkins-prod-pr", color: '#FF0000', @@ -127,4 +153,4 @@ timeout(time: global_timeout, unit: 'HOURS') { } } -// vi: ft=groovy +// vim: ft=groovy diff --git a/.ci/kitchen-debian9-py2 b/.ci/kitchen-debian9-py2 index 0b11c88725b..08480b710d6 100644 --- a/.ci/kitchen-debian9-py2 +++ b/.ci/kitchen-debian9-py2 @@ -4,6 +4,11 @@ def testrun_timeout = 6 // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; +def distro_name = 'debian' +def distro_version = '9' +def python_version = 'py2' +def golden_images_branch = '2018.3' + properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ @@ -16,101 +21,122 @@ properties([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) + timeout(time: global_timeout, unit: 'HOURS') { node('kitchen-slave') { - timestamps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + credentialsId: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', - 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'NOX_ENV_NAME=runtests-zeromq', - 'NOX_PASSTHROUGH_OPTS=--ssh-tests', - 'NOX_ENABLE_FROM_FILENAMES=true', - 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=debian9,py2', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py2', - 'TEST_PLATFORM=debian-9', - 'PY_COLORS=1', - "FORCE_FULL=${params.runFull}", - ]) { - // Set the GH status even before cloning the repo - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - // Checkout the repo - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { + timestamps { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', + 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'NOX_ENV_NAME=runtests-zeromq', + 'NOX_ENABLE_FROM_FILENAMES=true', + 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", + "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", + 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + "TEST_SUITE=${python_version}", + "TEST_PLATFORM=${distro_name}-${distro_version}", + "FORCE_FULL=${params.runFull}", + ]) { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + // Setup the kitchen required bundle stage('setup-bundle') { sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } - try { - stage('run kitchen') { - timeout(time: testrun_timeout, unit: 'HOURS') { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' - } - } - } - } - } finally { - stage('cleanup kitchen') { - script { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - } - } - archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' - } - stage('report code coverage') { - script { - withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { - sh ''' - if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then - curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" - fi - ''' - } - } + + stage('Create VM') { + retry(3) { + sh ''' + t=$(shuf -i 1-15 -n 1); echo "Sleeping $t seconds"; sleep $t + bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' } } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { try { - junit 'artifacts/xml-unittests-output/*.xml' + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem || ssh-add ~/.ssh/kitchen.pem' + try { + timeout(time: testrun_timeout, unit: 'HOURS') { + stage('Converge VM') { + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + } + stage('Run Tests') { + withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) { + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } + } + } finally { + try { + stage('Download Artefacts') { + withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){ + sh ''' + bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0 + ''' + } + } + archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + stage('Cleanup') { + sh ''' + bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' + } + stage('Upload Coverage') { + script { + withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { + sh ''' + if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then + curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" + fi + ''' + } + } + } + } + } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' } finally { cleanWs notFailBuild: true if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } try { slackSend channel: "#jenkins-prod-pr", color: '#FF0000', @@ -127,4 +153,4 @@ timeout(time: global_timeout, unit: 'HOURS') { } } -// vi: ft=groovy +// vim: ft=groovy diff --git a/.ci/kitchen-debian9-py3 b/.ci/kitchen-debian9-py3 index 4d47fef0c0a..1e5b194951e 100644 --- a/.ci/kitchen-debian9-py3 +++ b/.ci/kitchen-debian9-py3 @@ -4,6 +4,11 @@ def testrun_timeout = 6 // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; +def distro_name = 'debian' +def distro_version = '9' +def python_version = 'py3' +def golden_images_branch = '2018.3' + properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ @@ -16,101 +21,122 @@ properties([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) + timeout(time: global_timeout, unit: 'HOURS') { node('kitchen-slave') { - timestamps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + credentialsId: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', - 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'NOX_ENV_NAME=runtests-zeromq', - 'NOX_PASSTHROUGH_OPTS=--ssh-tests', - 'NOX_ENABLE_FROM_FILENAMES=true', - 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=debian9,py3', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py3', - 'TEST_PLATFORM=debian-9', - 'PY_COLORS=1', - "FORCE_FULL=${params.runFull}", - ]) { - // Set the GH status even before cloning the repo - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - // Checkout the repo - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { + timestamps { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', + 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'NOX_ENV_NAME=runtests-zeromq', + 'NOX_ENABLE_FROM_FILENAMES=true', + 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", + "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", + 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + "TEST_SUITE=${python_version}", + "TEST_PLATFORM=${distro_name}-${distro_version}", + "FORCE_FULL=${params.runFull}", + ]) { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + // Setup the kitchen required bundle stage('setup-bundle') { sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } - try { - stage('run kitchen') { - timeout(time: testrun_timeout, unit: 'HOURS') { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' - } - } - } - } - } finally { - stage('cleanup kitchen') { - script { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - } - } - archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' - } - stage('report code coverage') { - script { - withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { - sh ''' - if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then - curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" - fi - ''' - } - } + + stage('Create VM') { + retry(3) { + sh ''' + t=$(shuf -i 1-15 -n 1); echo "Sleeping $t seconds"; sleep $t + bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' } } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { try { - junit 'artifacts/xml-unittests-output/*.xml' + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem || ssh-add ~/.ssh/kitchen.pem' + try { + timeout(time: testrun_timeout, unit: 'HOURS') { + stage('Converge VM') { + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + } + stage('Run Tests') { + withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) { + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } + } + } finally { + try { + stage('Download Artefacts') { + withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){ + sh ''' + bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0 + ''' + } + } + archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + stage('Cleanup') { + sh ''' + bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' + } + stage('Upload Coverage') { + script { + withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { + sh ''' + if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then + curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" + fi + ''' + } + } + } + } + } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' } finally { cleanWs notFailBuild: true if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } try { slackSend channel: "#jenkins-prod-pr", color: '#FF0000', @@ -127,4 +153,4 @@ timeout(time: global_timeout, unit: 'HOURS') { } } -// vi: ft=groovy +// vim: ft=groovy diff --git a/.ci/kitchen-fedora29-py2 b/.ci/kitchen-fedora29-py2 index 4b959bc8276..b995a7e310c 100644 --- a/.ci/kitchen-fedora29-py2 +++ b/.ci/kitchen-fedora29-py2 @@ -4,6 +4,11 @@ def testrun_timeout = 6 // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; +def distro_name = 'fedora' +def distro_version = '29' +def python_version = 'py2' +def golden_images_branch = '2018.3' + properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ @@ -16,101 +21,122 @@ properties([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) + timeout(time: global_timeout, unit: 'HOURS') { node('kitchen-slave') { - timestamps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + credentialsId: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', - 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'NOX_ENV_NAME=runtests-zeromq', - 'NOX_PASSTHROUGH_OPTS=--ssh-tests', - 'NOX_ENABLE_FROM_FILENAMES=true', - 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=fedora29,py2', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py2', - 'TEST_PLATFORM=fedora-29', - 'PY_COLORS=1', - "FORCE_FULL=${params.runFull}", - ]) { - // Set the GH status even before cloning the repo - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - // Checkout the repo - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { + timestamps { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', + 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'NOX_ENV_NAME=runtests-zeromq', + 'NOX_ENABLE_FROM_FILENAMES=true', + 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", + "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", + 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + "TEST_SUITE=${python_version}", + "TEST_PLATFORM=${distro_name}-${distro_version}", + "FORCE_FULL=${params.runFull}", + ]) { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + // Setup the kitchen required bundle stage('setup-bundle') { sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } - try { - stage('run kitchen') { - timeout(time: testrun_timeout, unit: 'HOURS') { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' - } - } - } - } - } finally { - stage('cleanup kitchen') { - script { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - } - } - archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' - } - stage('report code coverage') { - script { - withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { - sh ''' - if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then - curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" - fi - ''' - } - } + + stage('Create VM') { + retry(3) { + sh ''' + t=$(shuf -i 1-15 -n 1); echo "Sleeping $t seconds"; sleep $t + bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' } } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { try { - junit 'artifacts/xml-unittests-output/*.xml' + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem || ssh-add ~/.ssh/kitchen.pem' + try { + timeout(time: testrun_timeout, unit: 'HOURS') { + stage('Converge VM') { + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + } + stage('Run Tests') { + withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) { + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } + } + } finally { + try { + stage('Download Artefacts') { + withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){ + sh ''' + bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0 + ''' + } + } + archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + stage('Cleanup') { + sh ''' + bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' + } + stage('Upload Coverage') { + script { + withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { + sh ''' + if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then + curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" + fi + ''' + } + } + } + } + } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' } finally { cleanWs notFailBuild: true if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } try { slackSend channel: "#jenkins-prod-pr", color: '#FF0000', @@ -127,4 +153,4 @@ timeout(time: global_timeout, unit: 'HOURS') { } } -// vi: ft=groovy +// vim: ft=groovy diff --git a/.ci/kitchen-fedora29-py3 b/.ci/kitchen-fedora29-py3 index ad6079e7e8c..d304c8e0e4f 100644 --- a/.ci/kitchen-fedora29-py3 +++ b/.ci/kitchen-fedora29-py3 @@ -4,6 +4,11 @@ def testrun_timeout = 6 // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; +def distro_name = 'fedora' +def distro_version = '29' +def python_version = 'py3' +def golden_images_branch = '2018.3' + properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ @@ -16,101 +21,122 @@ properties([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) + timeout(time: global_timeout, unit: 'HOURS') { node('kitchen-slave') { - timestamps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + credentialsId: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', - 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'NOX_ENV_NAME=runtests-zeromq', - 'NOX_PASSTHROUGH_OPTS=--ssh-tests', - 'NOX_ENABLE_FROM_FILENAMES=true', - 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=fedora29,py3', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py3', - 'TEST_PLATFORM=fedora-29', - 'PY_COLORS=1', - "FORCE_FULL=${params.runFull}", - ]) { - // Set the GH status even before cloning the repo - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - // Checkout the repo - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { + timestamps { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', + 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'NOX_ENV_NAME=runtests-zeromq', + 'NOX_ENABLE_FROM_FILENAMES=true', + 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", + "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", + 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + "TEST_SUITE=${python_version}", + "TEST_PLATFORM=${distro_name}-${distro_version}", + "FORCE_FULL=${params.runFull}", + ]) { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + // Setup the kitchen required bundle stage('setup-bundle') { sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } - try { - stage('run kitchen') { - timeout(time: testrun_timeout, unit: 'HOURS') { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' - } - } - } - } - } finally { - stage('cleanup kitchen') { - script { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - } - } - archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' - } - stage('report code coverage') { - script { - withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { - sh ''' - if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then - curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" - fi - ''' - } - } + + stage('Create VM') { + retry(3) { + sh ''' + t=$(shuf -i 1-15 -n 1); echo "Sleeping $t seconds"; sleep $t + bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' } } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { try { - junit 'artifacts/xml-unittests-output/*.xml' + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem || ssh-add ~/.ssh/kitchen.pem' + try { + timeout(time: testrun_timeout, unit: 'HOURS') { + stage('Converge VM') { + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + } + stage('Run Tests') { + withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) { + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } + } + } finally { + try { + stage('Download Artefacts') { + withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){ + sh ''' + bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0 + ''' + } + } + archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + stage('Cleanup') { + sh ''' + bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' + } + stage('Upload Coverage') { + script { + withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { + sh ''' + if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then + curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" + fi + ''' + } + } + } + } + } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' } finally { cleanWs notFailBuild: true if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } try { slackSend channel: "#jenkins-prod-pr", color: '#FF0000', @@ -127,4 +153,4 @@ timeout(time: global_timeout, unit: 'HOURS') { } } -// vi: ft=groovy +// vim: ft=groovy diff --git a/.ci/kitchen-ubuntu1604-py2 b/.ci/kitchen-ubuntu1604-py2 index e3b7b6dcee9..baee0fa1bec 100644 --- a/.ci/kitchen-ubuntu1604-py2 +++ b/.ci/kitchen-ubuntu1604-py2 @@ -1,9 +1,14 @@ -// Define the maximum time, in hours, that a test run should run for +//// Define the maximum time, in hours, that a test run should run for def testrun_timeout = 6 // Now define a global pipeline timeout. This is the test run timeout with one(1) additional // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; +def distro_name = 'ubuntu' +def distro_version = '1604' +def python_version = 'py2' +def golden_images_branch = '2018.3' + properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ @@ -16,101 +21,122 @@ properties([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) + timeout(time: global_timeout, unit: 'HOURS') { node('kitchen-slave') { - timestamps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + credentialsId: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', - 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'NOX_ENV_NAME=runtests-zeromq', - 'NOX_PASSTHROUGH_OPTS=--ssh-tests', - 'NOX_ENABLE_FROM_FILENAMES=true', - 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=ubuntu1604,py2', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py2', - 'TEST_PLATFORM=ubuntu-1604', - 'PY_COLORS=1', - "FORCE_FULL=${params.runFull}", - ]) { - // Set the GH status even before cloning the repo - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - // Checkout the repo - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { + timestamps { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', + 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'NOX_ENV_NAME=runtests-zeromq', + 'NOX_ENABLE_FROM_FILENAMES=true', + 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", + "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", + 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + "TEST_SUITE=${python_version}", + "TEST_PLATFORM=${distro_name}-${distro_version}", + "FORCE_FULL=${params.runFull}", + ]) { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + // Setup the kitchen required bundle stage('setup-bundle') { sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } - try { - stage('run kitchen') { - timeout(time: testrun_timeout, unit: 'HOURS') { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' - } - } - } - } - } finally { - stage('cleanup kitchen') { - script { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - } - } - archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' - } - stage('report code coverage') { - script { - withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { - sh ''' - if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then - curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" - fi - ''' - } - } + + stage('Create VM') { + retry(3) { + sh ''' + t=$(shuf -i 1-15 -n 1); echo "Sleeping $t seconds"; sleep $t + bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' } } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { try { - junit 'artifacts/xml-unittests-output/*.xml' + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem || ssh-add ~/.ssh/kitchen.pem' + try { + timeout(time: testrun_timeout, unit: 'HOURS') { + stage('Converge VM') { + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + } + stage('Run Tests') { + withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) { + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } + } + } finally { + try { + stage('Download Artefacts') { + withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){ + sh ''' + bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0 + ''' + } + } + archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + stage('Cleanup') { + sh ''' + bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' + } + stage('Upload Coverage') { + script { + withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { + sh ''' + if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then + curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" + fi + ''' + } + } + } + } + } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' } finally { cleanWs notFailBuild: true if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } try { slackSend channel: "#jenkins-prod-pr", color: '#FF0000', @@ -127,4 +153,4 @@ timeout(time: global_timeout, unit: 'HOURS') { } } -// vi: ft=groovy +// vim: ft=groovy diff --git a/.ci/kitchen-ubuntu1604-py3 b/.ci/kitchen-ubuntu1604-py3 index a8451473774..340b87f0ec4 100644 --- a/.ci/kitchen-ubuntu1604-py3 +++ b/.ci/kitchen-ubuntu1604-py3 @@ -1,9 +1,14 @@ -// Define the maximum time, in hours, that a test run should run for +//// Define the maximum time, in hours, that a test run should run for def testrun_timeout = 6 // Now define a global pipeline timeout. This is the test run timeout with one(1) additional // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; +def distro_name = 'ubuntu' +def distro_version = '1604' +def python_version = 'py3' +def golden_images_branch = '2018.3' + properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ @@ -16,101 +21,122 @@ properties([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) + timeout(time: global_timeout, unit: 'HOURS') { node('kitchen-slave') { - timestamps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + credentialsId: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', - 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'NOX_ENV_NAME=runtests-zeromq', - 'NOX_PASSTHROUGH_OPTS=--ssh-tests', - 'NOX_ENABLE_FROM_FILENAMES=true', - 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=ubuntu1604,py3', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py3', - 'TEST_PLATFORM=ubuntu-1604', - 'PY_COLORS=1', - "FORCE_FULL=${params.runFull}", - ]) { - // Set the GH status even before cloning the repo - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - // Checkout the repo - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { + timestamps { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', + 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'NOX_ENV_NAME=runtests-zeromq', + 'NOX_ENABLE_FROM_FILENAMES=true', + 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", + "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", + 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + "TEST_SUITE=${python_version}", + "TEST_PLATFORM=${distro_name}-${distro_version}", + "FORCE_FULL=${params.runFull}", + ]) { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + // Setup the kitchen required bundle stage('setup-bundle') { sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } - try { - stage('run kitchen') { - timeout(time: testrun_timeout, unit: 'HOURS') { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' - } - } - } - } - } finally { - stage('cleanup kitchen') { - script { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - } - } - archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' - } - stage('report code coverage') { - script { - withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { - sh ''' - if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then - curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" - fi - ''' - } - } + + stage('Create VM') { + retry(3) { + sh ''' + t=$(shuf -i 1-15 -n 1); echo "Sleeping $t seconds"; sleep $t + bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' } } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { try { - junit 'artifacts/xml-unittests-output/*.xml' + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem || ssh-add ~/.ssh/kitchen.pem' + try { + timeout(time: testrun_timeout, unit: 'HOURS') { + stage('Converge VM') { + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + } + stage('Run Tests') { + withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) { + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } + } + } finally { + try { + stage('Download Artefacts') { + withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){ + sh ''' + bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0 + ''' + } + } + archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + stage('Cleanup') { + sh ''' + bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' + } + stage('Upload Coverage') { + script { + withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { + sh ''' + if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then + curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" + fi + ''' + } + } + } + } + } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' } finally { cleanWs notFailBuild: true if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } try { slackSend channel: "#jenkins-prod-pr", color: '#FF0000', @@ -127,4 +153,4 @@ timeout(time: global_timeout, unit: 'HOURS') { } } -// vi: ft=groovy +// vim: ft=groovy diff --git a/.ci/kitchen-ubuntu1804-py2 b/.ci/kitchen-ubuntu1804-py2 index 6f397945d8b..2a4037d8a69 100644 --- a/.ci/kitchen-ubuntu1804-py2 +++ b/.ci/kitchen-ubuntu1804-py2 @@ -4,6 +4,11 @@ def testrun_timeout = 6 // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; +def distro_name = 'ubuntu' +def distro_version = '1804' +def python_version = 'py2' +def golden_images_branch = '2018.3' + properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ @@ -16,101 +21,122 @@ properties([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) + timeout(time: global_timeout, unit: 'HOURS') { node('kitchen-slave') { - timestamps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + credentialsId: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', - 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'NOX_ENV_NAME=runtests-zeromq', - 'NOX_PASSTHROUGH_OPTS=--ssh-tests', - 'NOX_ENABLE_FROM_FILENAMES=true', - 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=ubuntu1804,py2', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py2', - 'TEST_PLATFORM=ubuntu-1804', - 'PY_COLORS=1', - "FORCE_FULL=${params.runFull}", - ]) { - // Set the GH status even before cloning the repo - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - // Checkout the repo - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { + timestamps { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', + 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'NOX_ENV_NAME=runtests-zeromq', + 'NOX_ENABLE_FROM_FILENAMES=true', + 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", + "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", + 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + "TEST_SUITE=${python_version}", + "TEST_PLATFORM=${distro_name}-${distro_version}", + "FORCE_FULL=${params.runFull}", + ]) { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + // Setup the kitchen required bundle stage('setup-bundle') { sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } - try { - stage('run kitchen') { - timeout(time: testrun_timeout, unit: 'HOURS') { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' - } - } - } - } - } finally { - stage('cleanup kitchen') { - script { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - } - } - archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' - } - stage('report code coverage') { - script { - withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { - sh ''' - if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then - curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" - fi - ''' - } - } + + stage('Create VM') { + retry(3) { + sh ''' + t=$(shuf -i 1-15 -n 1); echo "Sleeping $t seconds"; sleep $t + bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' } } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { try { - junit 'artifacts/xml-unittests-output/*.xml' + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem || ssh-add ~/.ssh/kitchen.pem' + try { + timeout(time: testrun_timeout, unit: 'HOURS') { + stage('Converge VM') { + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + } + stage('Run Tests') { + withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) { + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } + } + } finally { + try { + stage('Download Artefacts') { + withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){ + sh ''' + bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0 + ''' + } + } + archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + stage('Cleanup') { + sh ''' + bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' + } + stage('Upload Coverage') { + script { + withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { + sh ''' + if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then + curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" + fi + ''' + } + } + } + } + } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' } finally { cleanWs notFailBuild: true if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } try { slackSend channel: "#jenkins-prod-pr", color: '#FF0000', @@ -127,4 +153,4 @@ timeout(time: global_timeout, unit: 'HOURS') { } } -// vi: ft=groovy +// vim: ft=groovy diff --git a/.ci/kitchen-ubuntu1804-py3 b/.ci/kitchen-ubuntu1804-py3 index eee089a5d99..abbb9cc4611 100644 --- a/.ci/kitchen-ubuntu1804-py3 +++ b/.ci/kitchen-ubuntu1804-py3 @@ -4,6 +4,11 @@ def testrun_timeout = 6 // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; +def distro_name = 'ubuntu' +def distro_version = '1804' +def python_version = 'py3' +def golden_images_branch = '2018.3' + properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ @@ -16,101 +21,122 @@ properties([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) + timeout(time: global_timeout, unit: 'HOURS') { node('kitchen-slave') { - timestamps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + credentialsId: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', - 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'NOX_ENV_NAME=runtests-zeromq', - 'NOX_PASSTHROUGH_OPTS=--ssh-tests', - 'NOX_ENABLE_FROM_FILENAMES=true', - 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=ubuntu1804,py3', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py3', - 'TEST_PLATFORM=ubuntu-1804', - 'PY_COLORS=1', - "FORCE_FULL=${params.runFull}", - ]) { - // Set the GH status even before cloning the repo - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - // Checkout the repo - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { + timestamps { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', + 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'NOX_ENV_NAME=runtests-zeromq', + 'NOX_ENABLE_FROM_FILENAMES=true', + 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", + "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", + 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + "TEST_SUITE=${python_version}", + "TEST_PLATFORM=${distro_name}-${distro_version}", + "FORCE_FULL=${params.runFull}", + ]) { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + // Setup the kitchen required bundle stage('setup-bundle') { sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } - try { - stage('run kitchen') { - timeout(time: testrun_timeout, unit: 'HOURS') { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' - } - } - } - } - } finally { - stage('cleanup kitchen') { - script { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - } - } - archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' - } - stage('report code coverage') { - script { - withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { - sh ''' - if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then - curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" - fi - ''' - } - } + + stage('Create VM') { + retry(3) { + sh ''' + t=$(shuf -i 1-15 -n 1); echo "Sleeping $t seconds"; sleep $t + bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' } } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { try { - junit 'artifacts/xml-unittests-output/*.xml' + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem || ssh-add ~/.ssh/kitchen.pem' + try { + timeout(time: testrun_timeout, unit: 'HOURS') { + stage('Converge VM') { + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + } + stage('Run Tests') { + withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) { + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } + } + } finally { + try { + stage('Download Artefacts') { + withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){ + sh ''' + bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0 + ''' + } + } + archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + stage('Cleanup') { + sh ''' + bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' + } + stage('Upload Coverage') { + script { + withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { + sh ''' + if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then + curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" + fi + ''' + } + } + } + } + } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' } finally { cleanWs notFailBuild: true if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } try { slackSend channel: "#jenkins-prod-pr", color: '#FF0000', @@ -127,4 +153,4 @@ timeout(time: global_timeout, unit: 'HOURS') { } } -// vi: ft=groovy +// vim: ft=groovy diff --git a/.ci/kitchen-windows2016-py2 b/.ci/kitchen-windows2016-py2 index f2f42c423b2..4a8cd2212d0 100644 --- a/.ci/kitchen-windows2016-py2 +++ b/.ci/kitchen-windows2016-py2 @@ -1,9 +1,14 @@ // Define the maximum time, in hours, that a test run should run for -def testrun_timeout = 8 +def testrun_timeout = 6 // Now define a global pipeline timeout. This is the test run timeout with one(1) additional // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; +def distro_name = 'windows' +def distro_version = '2016' +def python_version = 'py2' +def golden_images_branch = '2018.3' + properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ @@ -16,100 +21,122 @@ properties([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) + timeout(time: global_timeout, unit: 'HOURS') { node('kitchen-slave') { - timestamps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + credentialsId: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', - 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-win-verifier.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'NOX_ENV_NAME=runtests-zeromq', - 'NOX_ENABLE_FROM_FILENAMES=true', - 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=windows2016,py2', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py2', - 'TEST_PLATFORM=windows-2016', - 'PY_COLORS=1', - "FORCE_FULL=${params.runFull}", - ]) { - // Set the GH status even before cloning the repo - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - // Checkout the repo - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { + timestamps { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', + 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'NOX_ENV_NAME=runtests-zeromq', + 'NOX_ENABLE_FROM_FILENAMES=true', + 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", + "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", + 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + "TEST_SUITE=${python_version}", + "TEST_PLATFORM=${distro_name}-${distro_version}", + "FORCE_FULL=${params.runFull}", + ]) { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + // Setup the kitchen required bundle stage('setup-bundle') { sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } - try { - stage('run kitchen') { - timeout(time: testrun_timeout, unit: 'HOURS') { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' - } - } - } - } - } finally { - stage('cleanup kitchen') { - script { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - } - } - archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' - } - stage('report code coverage') { - script { - withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { - sh ''' - if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then - curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" - fi - ''' - } - } + + stage('Create VM') { + retry(3) { + sh ''' + t=$(shuf -i 1-15 -n 1); echo "Sleeping $t seconds"; sleep $t + bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' } } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { try { - junit 'artifacts/xml-unittests-output/*.xml' + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem || ssh-add ~/.ssh/kitchen.pem' + try { + timeout(time: testrun_timeout, unit: 'HOURS') { + stage('Converge VM') { + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + } + stage('Run Tests') { + withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) { + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } + } + } finally { + try { + stage('Download Artefacts') { + withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){ + sh ''' + bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0 + ''' + } + } + archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + stage('Cleanup') { + sh ''' + bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' + } + stage('Upload Coverage') { + script { + withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { + sh ''' + if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then + curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" + fi + ''' + } + } + } + } + } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' } finally { cleanWs notFailBuild: true if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } try { slackSend channel: "#jenkins-prod-pr", color: '#FF0000', @@ -126,4 +153,4 @@ timeout(time: global_timeout, unit: 'HOURS') { } } -// vi: ft=groovy +// vim: ft=groovy diff --git a/.ci/kitchen-windows2016-py3 b/.ci/kitchen-windows2016-py3 index 73413e51076..80413adcd63 100644 --- a/.ci/kitchen-windows2016-py3 +++ b/.ci/kitchen-windows2016-py3 @@ -1,9 +1,14 @@ // Define the maximum time, in hours, that a test run should run for -def testrun_timeout = 8 +def testrun_timeout = 6 // Now define a global pipeline timeout. This is the test run timeout with one(1) additional // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; +def distro_name = 'windows' +def distro_version = '2016' +def python_version = 'py3' +def golden_images_branch = '2018.3' + properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), [ @@ -16,100 +21,121 @@ properties([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) ]) + timeout(time: global_timeout, unit: 'HOURS') { node('kitchen-slave') { - timestamps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + credentialsId: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', - 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-win-verifier.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'NOX_ENV_NAME=runtests-zeromq', - 'NOX_ENABLE_FROM_FILENAMES=true', - 'GOLDEN_IMAGES_CI_BRANCH=2018.3', - 'CODECOV_FLAGS=windows2016,py3', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py3', - 'TEST_PLATFORM=windows-2016', - 'PY_COLORS=1', - "FORCE_FULL=${params.runFull}", - ]) { - // Set the GH status even before cloning the repo - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - // Checkout the repo - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { + timestamps { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', + 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'NOX_ENV_NAME=runtests-zeromq', + 'NOX_ENABLE_FROM_FILENAMES=true', + "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", + "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", + 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + "TEST_SUITE=${python_version}", + "TEST_PLATFORM=${distro_name}-${distro_version}", + "FORCE_FULL=${params.runFull}", + ]) { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + // Setup the kitchen required bundle stage('setup-bundle') { sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } - try { - stage('run kitchen') { - timeout(time: testrun_timeout, unit: 'HOURS') { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' - } - } - } - } - } finally { - stage('cleanup kitchen') { - script { - withCredentials([ - [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] - ]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - } - } - archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' - } - stage('report code coverage') { - script { - withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { - sh ''' - if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then - curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" - fi - ''' - } - } + + stage('Create VM') { + retry(3) { + sh ''' + t=$(shuf -i 1-15 -n 1); echo "Sleeping $t seconds"; sleep $t + bundle exec kitchen create $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' } } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { try { - junit 'artifacts/xml-unittests-output/*.xml' + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem || ssh-add ~/.ssh/kitchen.pem' + try { + timeout(time: testrun_timeout, unit: 'HOURS') { + stage('Converge VM') { + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + } + stage('Run Tests') { + withEnv(["DONT_DOWNLOAD_ARTEFACTS=1"]) { + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } + } + } finally { + try { + stage('Download Artefacts') { + withEnv(["ONLY_DOWNLOAD_ARTEFACTS=1"]){ + sh ''' + bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM || exit 0 + ''' + } + } + archiveArtifacts artifacts: 'artifacts/*,artifacts/**/*' + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + stage('Cleanup') { + sh ''' + bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM; echo "ExitCode: $?;" + ''' + } + stage('Upload Coverage') { + script { + withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) { + sh ''' + if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then + curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" + fi + ''' + } + } + } + } + } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' } finally { cleanWs notFailBuild: true if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } try { slackSend channel: "#jenkins-prod-pr", color: '#FF0000', @@ -126,4 +152,4 @@ timeout(time: global_timeout, unit: 'HOURS') { } } -// vi: ft=groovy +// vim: ft=groovy diff --git a/.ci/lint b/.ci/lint index 7fdd4c3ab32..3888fb9817a 100644 --- a/.ci/lint +++ b/.ci/lint @@ -1,176 +1,195 @@ -pipeline { - agent { label 'pr-lint-slave' } - options { - timestamps() - ansiColor('xterm') - timeout(time: 3, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '10')) - } - environment { - PYENV_ROOT = "/usr/local/pyenv" - PATH = "$PYENV_ROOT/bin:$PATH" - PY_COLORS = 1 - } - stages { - stage('github-pending') { - steps { - githubNotify credentialsId: 'test-jenkins-credentials', - description: 'Python lint on changes begins...', - status: 'PENDING', - context: "jenkins/pr/lint" - } - } - stage('setup') { - steps { - sh ''' - # Need -M to detect renames otherwise they are reported as Delete and Add, need -C to detect copies, -C includes -M - # -M is on by default in git 2.9+ - git diff --name-status -l99999 -C "origin/$CHANGE_TARGET" > file-list-status.log - # the -l increase the search limit, lets use awk so we do not need to repeat the search above. - gawk 'BEGIN {FS="\\t"} {if ($1 != "D") {print $NF}}' file-list-status.log > file-list-changed.log - gawk 'BEGIN {FS="\\t"} {if ($1 == "D") {print $NF}}' file-list-status.log > file-list-deleted.log - (git diff --name-status -l99999 -C "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME";echo "---";git diff --name-status -l99999 -C "origin/$BRANCH_NAME";printenv|grep -E '=[0-9a-z]{40,}+$|COMMIT=|BRANCH') > file-list-experiment.log - eval "$(pyenv init -)" - pyenv --version - pyenv install --skip-existing 2.7.15 - pyenv shell 2.7.15 - python --version - pip install -U https://github.com/s0undt3ch/nox/archive/hotfix/py2.zip#egg=Nox==2018.10.17 - nox --version - # Create the required virtualens in serial - nox --install-only -e lint-salt - nox --install-only -e lint-tests - ''' - archiveArtifacts artifacts: 'file-list-status.log,file-list-changed.log,file-list-deleted.log,file-list-experiment.log' - } - } - stage('linting chg') { - parallel { - stage('lint salt chg') { - when { - expression { return readFile('file-list-changed.log') =~ /(?i)(^|\n)(salt\/.*\.py|setup\.py)\n/ } +// Define the maximum time, in hours, that a test run should run for +def global_timeout = 3 + +properties([ + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), + [ + $class: 'ScannerJobProperty', doNotScan: false + ], + [ + $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false + ], +]) + +def shell_header + +timeout(time: global_timeout, unit: 'HOURS') { + node('lint') { + ansiColor('xterm') { + timestamps { + try { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: 'Python lint begins...', + status: 'PENDING', + context: "jenkins/pr/lint" + } + shell_header = 'export PYENV_ROOT="/usr/local/pyenv"\nexport PATH="$PYENV_ROOT/bin:$PATH"' + } else { + shell_header = '' } - steps { - sh ''' - eval "$(pyenv init - --no-rehash)" + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + + // Setup the kitchen required bundle + stage('Setup') { + sh shell_header + ''' + # Need -M to detect renames otherwise they are reported as Delete and Add, need -C to detect copies, -C includes -M + # -M is on by default in git 2.9+ + git diff --name-status -l99999 -C "origin/$CHANGE_TARGET" > file-list-status.log + # the -l increase the search limit, lets use awk so we do not need to repeat the search above. + gawk 'BEGIN {FS="\\t"} {if ($1 != "D") {print $NF}}' file-list-status.log > file-list-changed.log + gawk 'BEGIN {FS="\\t"} {if ($1 == "D") {print $NF}}' file-list-status.log > file-list-deleted.log + (git diff --name-status -l99999 -C "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME";echo "---";git diff --name-status -l99999 -C "origin/$BRANCH_NAME";printenv|grep -E '=[0-9a-z]{40,}+$|COMMIT=|BRANCH') > file-list-experiment.log + eval "$(pyenv init -)" + pyenv --version + pyenv install --skip-existing 2.7.15 pyenv shell 2.7.15 - EC=254 - export PYLINT_REPORT=pylint-report-salt-chg.log - grep -Ei '^salt/.*\\.py$|^setup\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' nox -e lint-salt -- - EC=$? - exit $EC + python --version + pip install -U https://github.com/s0undt3ch/nox/archive/hotfix/py2.zip#egg=Nox==2018.10.17 + nox --version + # Create the required virtualenvs in serial + nox --install-only -e lint-salt + nox --install-only -e lint-tests ''' + archiveArtifacts artifacts: 'file-list-status.log,file-list-changed.log,file-list-deleted.log,file-list-experiment.log' } - } - stage('lint test chg') { - when { - expression { return readFile('file-list-changed.log') =~ /(?i)(^|\n)tests\/.*\.py\n/ } + stage('Lint Changes') { + try { + parallel( + lintSalt: { + stage('Lint Salt Changes') { + if (readFile('file-list-changed.log') =~ /(?i)(^|\n)(salt\/.*\.py|setup\.py)\n/) { + sh shell_header + ''' + eval "$(pyenv init - --no-rehash)" + pyenv shell 2.7.15 + EC=254 + export PYLINT_REPORT=pylint-report-salt-chg.log + grep -Ei '^salt/.*\\.py$|^setup\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' nox -e lint-salt -- + EC=$? + exit $EC + ''' + } + } + }, + lintTests: { + stage('Lint Test Changes') { + if (readFile('file-list-changed.log') =~ /(?i)(^|\n)tests\/.*\.py\n/) { + sh shell_header + ''' + eval "$(pyenv init - --no-rehash)" + pyenv shell 2.7.15 + EC=254 + export PYLINT_REPORT=pylint-report-tests-chg.log + grep -Ei '^tests/.*\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' nox -e lint-tests -- + EC=$? + exit $EC + ''' + } + } + } + ) + } finally { + archiveArtifacts artifacts: 'pylint-report-*-chg.log', allowEmptyArchive: true + step([$class: 'WarningsPublisher', + parserConfigurations: [[ + parserName: 'PyLint', + pattern: 'pylint-report-*-chg.log' + ]], + failedTotalAll: '0', + useDeltaValues: false, + canRunOnFailed: true, + usePreviousBuildAsReference: true + ]) + } } - steps { - sh ''' - eval "$(pyenv init - --no-rehash)" - pyenv shell 2.7.15 - EC=254 - export PYLINT_REPORT=pylint-report-tests-chg.log - grep -Ei '^tests/.*\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' nox -e lint-tests -- - EC=$? - exit $EC - ''' + stage('Lint Full') { + if (env.CHANGE_BRANCH =~ /(?i)^merge[._-]/) { + // perform a full linit if this is a merge forward and the change only lint passed. + try { + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: 'Python lint on everything begins...', + status: 'PENDING', + context: "jenkins/pr/lint" + } + parallel( + lintSaltFull: { + stage('Lint Salt Full') { + sh shell_header + ''' + eval "$(pyenv init - --no-rehash)" + pyenv shell 2.7.15 + EC=254 + export PYLINT_REPORT=pylint-report-salt-full.log + nox -e lint-salt + EC=$? + exit $EC + ''' + } + }, + lintTestsFull: { + stage('Lint Tests Full') { + sh shell_header + ''' + eval "$(pyenv init - --no-rehash)" + pyenv shell 2.7.15 + EC=254 + export PYLINT_REPORT=pylint-report-tests-full.log + nox -e lint-salt + EC=$? + exit $EC + ''' + } + } + ) + } finally { + archiveArtifacts artifacts: 'pylint-report-*-full.log', allowEmptyArchive: true + step([$class: 'WarningsPublisher', + parserConfigurations: [[ + parserName: 'PyLint', + pattern: 'pylint-report-*-full.log' + ]], + failedTotalAll: '0', + useDeltaValues: false, + canRunOnFailed: true, + usePreviousBuildAsReference: true + ]) + } + } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' + } finally { + cleanWs notFailBuild: true + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: 'The lint test passed', + status: 'SUCCESS', + context: "jenkins/pr/lint" + } + } else { + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: 'The lint test failed', + status: 'FAILURE', + context: "jenkins/pr/lint" + } + try { + slackSend channel: "#jenkins-prod-pr", + color: '#FF0000', + message: "FAILED: PR-Job: '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" + } catch (Exception e) { + sh 'echo Failed to send the Slack notification' + } } } } - post { - always { - archiveArtifacts artifacts: 'pylint-report-*-chg.log', allowEmptyArchive: true - step([$class: 'WarningsPublisher', - parserConfigurations: [[ - parserName: 'PyLint', - pattern: 'pylint-report-*-chg.log' - ]], - failedTotalAll: '0', - useDeltaValues: false, - canRunOnFailed: true, - usePreviousBuildAsReference: true - ]) - } - } - } - stage('linting all') { - // perform a full linit if this is a merge forward and the change only lint passed. - when { - expression { return env.CHANGE_BRANCH =~ /(?i)^merge[._-]/ } - } - parallel { - stage('setup full') { - steps { - githubNotify credentialsId: 'test-jenkins-credentials', - description: 'Python lint on everything begins...', - status: 'PENDING', - context: "jenkins/pr/lint" - } - } - stage('lint salt full') { - steps { - sh ''' - eval "$(pyenv init - --no-rehash)" - pyenv shell 2.7.15 - EC=254 - export PYLINT_REPORT=pylint-report-salt-full.log - nox -e lint-salt - EC=$? - exit $EC - ''' - } - } - stage('lint test full') { - steps { - sh ''' - eval "$(pyenv init - --no-rehash)" - pyenv shell 2.7.15 - EC=254 - export PYLINT_REPORT=pylint-report-tests-full.log - nox -e lint-salt - EC=$? - exit $EC - ''' - } - } - } - post { - always { - archiveArtifacts artifacts: 'pylint-report-*-full.log', allowEmptyArchive: true - step([$class: 'WarningsPublisher', - parserConfigurations: [[ - parserName: 'PyLint', - pattern: 'pylint-report-*-full.log' - ]], - failedTotalAll: '0', - useDeltaValues: false, - canRunOnFailed: true, - usePreviousBuildAsReference: true - ]) - } - } - } - } - post { - always { - cleanWs() - } - success { - githubNotify credentialsId: 'test-jenkins-credentials', - description: 'Python lint test has passed', - status: 'SUCCESS', - context: "jenkins/pr/lint" - } - failure { - githubNotify credentialsId: 'test-jenkins-credentials', - description: 'Python lint test has failed', - status: 'FAILURE', - context: "jenkins/pr/lint" - slackSend channel: "#jenkins-prod-pr", - color: '#FF0000', - message: "FAILED: PR-Job: '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" } } } + +// vim: ft=groovy From 662efb5255f1f1f8e0406d8dd32c6920501c181b Mon Sep 17 00:00:00 2001 From: Bryce Larson Date: Wed, 19 Jun 2019 23:01:45 -0600 Subject: [PATCH 30/31] CHANGE_TARGET isn't always available, hard coding replacement --- .ci/kitchen-centos6-py2 | 4 +- .ci/kitchen-centos7-py2 | 4 +- .ci/kitchen-centos7-py3 | 4 +- .ci/kitchen-debian8-py2 | 4 +- .ci/kitchen-debian8-py3 | 4 +- .ci/kitchen-debian9-py2 | 4 +- .ci/kitchen-debian9-py3 | 4 +- .ci/kitchen-fedora29-py2 | 4 +- .ci/kitchen-fedora29-py3 | 4 +- .ci/kitchen-ubuntu1604-py2 | 4 +- .ci/kitchen-ubuntu1604-py3 | 4 +- .ci/kitchen-ubuntu1804-py2 | 4 +- .ci/kitchen-ubuntu1804-py3 | 4 +- .ci/kitchen-windows2016-py2 | 4 +- .ci/kitchen-windows2016-py3 | 4 +- .ci/lint | 294 ++++++++++++++++++------------------ 16 files changed, 195 insertions(+), 159 deletions(-) diff --git a/.ci/kitchen-centos6-py2 b/.ci/kitchen-centos6-py2 index e0212799ca3..7d88c8d52ae 100644 --- a/.ci/kitchen-centos6-py2 +++ b/.ci/kitchen-centos6-py2 @@ -8,6 +8,7 @@ def distro_name = 'centos' def distro_version = '6' def python_version = 'py2' def golden_images_branch = '2018.3' +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), @@ -34,6 +35,7 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', @@ -62,7 +64,7 @@ timeout(time: global_timeout, unit: 'HOURS') { // Setup the kitchen required bundle stage('setup-bundle') { - sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } diff --git a/.ci/kitchen-centos7-py2 b/.ci/kitchen-centos7-py2 index 809d91db22e..e744c216106 100644 --- a/.ci/kitchen-centos7-py2 +++ b/.ci/kitchen-centos7-py2 @@ -8,6 +8,7 @@ def distro_name = 'centos' def distro_version = '7' def python_version = 'py2' def golden_images_branch = '2018.3' +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), @@ -34,6 +35,7 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', @@ -62,7 +64,7 @@ timeout(time: global_timeout, unit: 'HOURS') { // Setup the kitchen required bundle stage('setup-bundle') { - sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } diff --git a/.ci/kitchen-centos7-py3 b/.ci/kitchen-centos7-py3 index b6895823547..a1ca8e4740d 100644 --- a/.ci/kitchen-centos7-py3 +++ b/.ci/kitchen-centos7-py3 @@ -8,6 +8,7 @@ def distro_name = 'centos' def distro_version = '7' def python_version = 'py3' def golden_images_branch = '2018.3' +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), @@ -34,6 +35,7 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', @@ -62,7 +64,7 @@ timeout(time: global_timeout, unit: 'HOURS') { // Setup the kitchen required bundle stage('setup-bundle') { - sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } diff --git a/.ci/kitchen-debian8-py2 b/.ci/kitchen-debian8-py2 index 83418d261fb..fc95515acb6 100644 --- a/.ci/kitchen-debian8-py2 +++ b/.ci/kitchen-debian8-py2 @@ -8,6 +8,7 @@ def distro_name = 'debian' def distro_version = '8' def python_version = 'py2' def golden_images_branch = '2018.3' +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), @@ -34,6 +35,7 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', @@ -62,7 +64,7 @@ timeout(time: global_timeout, unit: 'HOURS') { // Setup the kitchen required bundle stage('setup-bundle') { - sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } diff --git a/.ci/kitchen-debian8-py3 b/.ci/kitchen-debian8-py3 index 371fdfdf6c8..465308952bf 100644 --- a/.ci/kitchen-debian8-py3 +++ b/.ci/kitchen-debian8-py3 @@ -8,6 +8,7 @@ def distro_name = 'debian' def distro_version = '8' def python_version = 'py3' def golden_images_branch = '2018.3' +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), @@ -34,6 +35,7 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', @@ -62,7 +64,7 @@ timeout(time: global_timeout, unit: 'HOURS') { // Setup the kitchen required bundle stage('setup-bundle') { - sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } diff --git a/.ci/kitchen-debian9-py2 b/.ci/kitchen-debian9-py2 index 08480b710d6..819d75a07bb 100644 --- a/.ci/kitchen-debian9-py2 +++ b/.ci/kitchen-debian9-py2 @@ -8,6 +8,7 @@ def distro_name = 'debian' def distro_version = '9' def python_version = 'py2' def golden_images_branch = '2018.3' +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), @@ -34,6 +35,7 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', @@ -62,7 +64,7 @@ timeout(time: global_timeout, unit: 'HOURS') { // Setup the kitchen required bundle stage('setup-bundle') { - sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } diff --git a/.ci/kitchen-debian9-py3 b/.ci/kitchen-debian9-py3 index 1e5b194951e..2a125497608 100644 --- a/.ci/kitchen-debian9-py3 +++ b/.ci/kitchen-debian9-py3 @@ -8,6 +8,7 @@ def distro_name = 'debian' def distro_version = '9' def python_version = 'py3' def golden_images_branch = '2018.3' +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), @@ -34,6 +35,7 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', @@ -62,7 +64,7 @@ timeout(time: global_timeout, unit: 'HOURS') { // Setup the kitchen required bundle stage('setup-bundle') { - sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } diff --git a/.ci/kitchen-fedora29-py2 b/.ci/kitchen-fedora29-py2 index b995a7e310c..b713814a4f4 100644 --- a/.ci/kitchen-fedora29-py2 +++ b/.ci/kitchen-fedora29-py2 @@ -8,6 +8,7 @@ def distro_name = 'fedora' def distro_version = '29' def python_version = 'py2' def golden_images_branch = '2018.3' +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), @@ -34,6 +35,7 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', @@ -62,7 +64,7 @@ timeout(time: global_timeout, unit: 'HOURS') { // Setup the kitchen required bundle stage('setup-bundle') { - sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } diff --git a/.ci/kitchen-fedora29-py3 b/.ci/kitchen-fedora29-py3 index d304c8e0e4f..0b222535d6e 100644 --- a/.ci/kitchen-fedora29-py3 +++ b/.ci/kitchen-fedora29-py3 @@ -8,6 +8,7 @@ def distro_name = 'fedora' def distro_version = '29' def python_version = 'py3' def golden_images_branch = '2018.3' +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), @@ -34,6 +35,7 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', @@ -62,7 +64,7 @@ timeout(time: global_timeout, unit: 'HOURS') { // Setup the kitchen required bundle stage('setup-bundle') { - sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } diff --git a/.ci/kitchen-ubuntu1604-py2 b/.ci/kitchen-ubuntu1604-py2 index baee0fa1bec..04bd927b2e8 100644 --- a/.ci/kitchen-ubuntu1604-py2 +++ b/.ci/kitchen-ubuntu1604-py2 @@ -8,6 +8,7 @@ def distro_name = 'ubuntu' def distro_version = '1604' def python_version = 'py2' def golden_images_branch = '2018.3' +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), @@ -34,6 +35,7 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', @@ -62,7 +64,7 @@ timeout(time: global_timeout, unit: 'HOURS') { // Setup the kitchen required bundle stage('setup-bundle') { - sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } diff --git a/.ci/kitchen-ubuntu1604-py3 b/.ci/kitchen-ubuntu1604-py3 index 340b87f0ec4..67efac98b70 100644 --- a/.ci/kitchen-ubuntu1604-py3 +++ b/.ci/kitchen-ubuntu1604-py3 @@ -8,6 +8,7 @@ def distro_name = 'ubuntu' def distro_version = '1604' def python_version = 'py3' def golden_images_branch = '2018.3' +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), @@ -34,6 +35,7 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', @@ -62,7 +64,7 @@ timeout(time: global_timeout, unit: 'HOURS') { // Setup the kitchen required bundle stage('setup-bundle') { - sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } diff --git a/.ci/kitchen-ubuntu1804-py2 b/.ci/kitchen-ubuntu1804-py2 index 2a4037d8a69..3bf433abf0b 100644 --- a/.ci/kitchen-ubuntu1804-py2 +++ b/.ci/kitchen-ubuntu1804-py2 @@ -8,6 +8,7 @@ def distro_name = 'ubuntu' def distro_version = '1804' def python_version = 'py2' def golden_images_branch = '2018.3' +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), @@ -34,6 +35,7 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', @@ -62,7 +64,7 @@ timeout(time: global_timeout, unit: 'HOURS') { // Setup the kitchen required bundle stage('setup-bundle') { - sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } diff --git a/.ci/kitchen-ubuntu1804-py3 b/.ci/kitchen-ubuntu1804-py3 index abbb9cc4611..b39748450af 100644 --- a/.ci/kitchen-ubuntu1804-py3 +++ b/.ci/kitchen-ubuntu1804-py3 @@ -8,6 +8,7 @@ def distro_name = 'ubuntu' def distro_version = '1804' def python_version = 'py3' def golden_images_branch = '2018.3' +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), @@ -34,6 +35,7 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', @@ -62,7 +64,7 @@ timeout(time: global_timeout, unit: 'HOURS') { // Setup the kitchen required bundle stage('setup-bundle') { - sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } diff --git a/.ci/kitchen-windows2016-py2 b/.ci/kitchen-windows2016-py2 index 4a8cd2212d0..8637b711d44 100644 --- a/.ci/kitchen-windows2016-py2 +++ b/.ci/kitchen-windows2016-py2 @@ -8,6 +8,7 @@ def distro_name = 'windows' def distro_version = '2016' def python_version = 'py2' def golden_images_branch = '2018.3' +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), @@ -34,6 +35,7 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', @@ -62,7 +64,7 @@ timeout(time: global_timeout, unit: 'HOURS') { // Setup the kitchen required bundle stage('setup-bundle') { - sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } diff --git a/.ci/kitchen-windows2016-py3 b/.ci/kitchen-windows2016-py3 index 80413adcd63..bb5fbb05f97 100644 --- a/.ci/kitchen-windows2016-py3 +++ b/.ci/kitchen-windows2016-py3 @@ -8,6 +8,7 @@ def distro_name = 'windows' def distro_version = '2016' def python_version = 'py3' def golden_images_branch = '2018.3' +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), @@ -34,6 +35,7 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", @@ -61,7 +63,7 @@ timeout(time: global_timeout, unit: 'HOURS') { // Setup the kitchen required bundle stage('setup-bundle') { - sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/${CHANGE_TARGET}' + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}' sh 'bundle install --with ec2 windows --without docker macos opennebula vagrant' } diff --git a/.ci/lint b/.ci/lint index 3888fb9817a..bf0ffe083db 100644 --- a/.ci/lint +++ b/.ci/lint @@ -12,145 +12,98 @@ properties([ ]) def shell_header +def salt_target_branch = '2018.3' timeout(time: global_timeout, unit: 'HOURS') { node('lint') { ansiColor('xterm') { timestamps { - try { - // Set the GH status even before cloning the repo - if (env.NODE_NAME.startsWith('jenkins-pr-')) { - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: 'Python lint begins...', - status: 'PENDING', - context: "jenkins/pr/lint" + withEnv([ + "SALT_TARGET_BRANCH=${salt_target_branch}", + ]) { + try { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: 'Python lint begins...', + status: 'PENDING', + context: "jenkins/pr/lint" + } + shell_header = 'export PYENV_ROOT="/usr/local/pyenv"\nexport PATH="$PYENV_ROOT/bin:$PATH"' + } else { + shell_header = '' + } + // Checkout the repo + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm } - shell_header = 'export PYENV_ROOT="/usr/local/pyenv"\nexport PATH="$PYENV_ROOT/bin:$PATH"' - } else { - shell_header = '' - } - // Checkout the repo - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - // Setup the kitchen required bundle - stage('Setup') { - sh shell_header + ''' - # Need -M to detect renames otherwise they are reported as Delete and Add, need -C to detect copies, -C includes -M - # -M is on by default in git 2.9+ - git diff --name-status -l99999 -C "origin/$CHANGE_TARGET" > file-list-status.log - # the -l increase the search limit, lets use awk so we do not need to repeat the search above. - gawk 'BEGIN {FS="\\t"} {if ($1 != "D") {print $NF}}' file-list-status.log > file-list-changed.log - gawk 'BEGIN {FS="\\t"} {if ($1 == "D") {print $NF}}' file-list-status.log > file-list-deleted.log - (git diff --name-status -l99999 -C "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME";echo "---";git diff --name-status -l99999 -C "origin/$BRANCH_NAME";printenv|grep -E '=[0-9a-z]{40,}+$|COMMIT=|BRANCH') > file-list-experiment.log - eval "$(pyenv init -)" - pyenv --version - pyenv install --skip-existing 2.7.15 - pyenv shell 2.7.15 - python --version - pip install -U https://github.com/s0undt3ch/nox/archive/hotfix/py2.zip#egg=Nox==2018.10.17 - nox --version - # Create the required virtualenvs in serial - nox --install-only -e lint-salt - nox --install-only -e lint-tests - ''' - archiveArtifacts artifacts: 'file-list-status.log,file-list-changed.log,file-list-deleted.log,file-list-experiment.log' - } - stage('Lint Changes') { - try { - parallel( - lintSalt: { - stage('Lint Salt Changes') { - if (readFile('file-list-changed.log') =~ /(?i)(^|\n)(salt\/.*\.py|setup\.py)\n/) { - sh shell_header + ''' - eval "$(pyenv init - --no-rehash)" - pyenv shell 2.7.15 - EC=254 - export PYLINT_REPORT=pylint-report-salt-chg.log - grep -Ei '^salt/.*\\.py$|^setup\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' nox -e lint-salt -- - EC=$? - exit $EC - ''' - } - } - }, - lintTests: { - stage('Lint Test Changes') { - if (readFile('file-list-changed.log') =~ /(?i)(^|\n)tests\/.*\.py\n/) { - sh shell_header + ''' - eval "$(pyenv init - --no-rehash)" - pyenv shell 2.7.15 - EC=254 - export PYLINT_REPORT=pylint-report-tests-chg.log - grep -Ei '^tests/.*\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' nox -e lint-tests -- - EC=$? - exit $EC - ''' - } - } - } - ) - } finally { - archiveArtifacts artifacts: 'pylint-report-*-chg.log', allowEmptyArchive: true - step([$class: 'WarningsPublisher', - parserConfigurations: [[ - parserName: 'PyLint', - pattern: 'pylint-report-*-chg.log' - ]], - failedTotalAll: '0', - useDeltaValues: false, - canRunOnFailed: true, - usePreviousBuildAsReference: true - ]) + // Setup the kitchen required bundle + stage('Setup') { + sh shell_header + ''' + git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH} + # Need -M to detect renames otherwise they are reported as Delete and Add, need -C to detect copies, -C includes -M + # -M is on by default in git 2.9+ + git diff --name-status -l99999 -C "origin/${SALT_TARGET_BRANCH}" > file-list-status.log + # the -l increase the search limit, lets use awk so we do not need to repeat the search above. + gawk 'BEGIN {FS="\\t"} {if ($1 != "D") {print $NF}}' file-list-status.log > file-list-changed.log + gawk 'BEGIN {FS="\\t"} {if ($1 == "D") {print $NF}}' file-list-status.log > file-list-deleted.log + (git diff --name-status -l99999 -C "origin/${SALT_TARGET_BRANCH}" "origin/$BRANCH_NAME";echo "---";git diff --name-status -l99999 -C "origin/$BRANCH_NAME";printenv|grep -E '=[0-9a-z]{40,}+$|COMMIT=|BRANCH') > file-list-experiment.log + eval "$(pyenv init -)" + pyenv --version + pyenv install --skip-existing 2.7.15 + pyenv shell 2.7.15 + python --version + pip install -U https://github.com/s0undt3ch/nox/archive/hotfix/py2.zip#egg=Nox==2018.10.17 + nox --version + # Create the required virtualenvs in serial + nox --install-only -e lint-salt + nox --install-only -e lint-tests + ''' + archiveArtifacts artifacts: 'file-list-status.log,file-list-changed.log,file-list-deleted.log,file-list-experiment.log' } - } - stage('Lint Full') { - if (env.CHANGE_BRANCH =~ /(?i)^merge[._-]/) { - // perform a full linit if this is a merge forward and the change only lint passed. + stage('Lint Changes') { try { - if (env.NODE_NAME.startsWith('jenkins-pr-')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: 'Python lint on everything begins...', - status: 'PENDING', - context: "jenkins/pr/lint" - } parallel( - lintSaltFull: { - stage('Lint Salt Full') { - sh shell_header + ''' - eval "$(pyenv init - --no-rehash)" - pyenv shell 2.7.15 - EC=254 - export PYLINT_REPORT=pylint-report-salt-full.log - nox -e lint-salt - EC=$? - exit $EC - ''' + lintSalt: { + stage('Lint Salt Changes') { + if (readFile('file-list-changed.log') =~ /(?i)(^|\n)(salt\/.*\.py|setup\.py)\n/) { + sh shell_header + ''' + eval "$(pyenv init - --no-rehash)" + pyenv shell 2.7.15 + EC=254 + export PYLINT_REPORT=pylint-report-salt-chg.log + grep -Ei '^salt/.*\\.py$|^setup\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' nox -e lint-salt -- + EC=$? + exit $EC + ''' + } } }, - lintTestsFull: { - stage('Lint Tests Full') { - sh shell_header + ''' - eval "$(pyenv init - --no-rehash)" - pyenv shell 2.7.15 - EC=254 - export PYLINT_REPORT=pylint-report-tests-full.log - nox -e lint-salt - EC=$? - exit $EC - ''' + lintTests: { + stage('Lint Test Changes') { + if (readFile('file-list-changed.log') =~ /(?i)(^|\n)tests\/.*\.py\n/) { + sh shell_header + ''' + eval "$(pyenv init - --no-rehash)" + pyenv shell 2.7.15 + EC=254 + export PYLINT_REPORT=pylint-report-tests-chg.log + grep -Ei '^tests/.*\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' nox -e lint-tests -- + EC=$? + exit $EC + ''' + } } } ) } finally { - archiveArtifacts artifacts: 'pylint-report-*-full.log', allowEmptyArchive: true + archiveArtifacts artifacts: 'pylint-report-*-chg.log', allowEmptyArchive: true step([$class: 'WarningsPublisher', parserConfigurations: [[ parserName: 'PyLint', - pattern: 'pylint-report-*-full.log' + pattern: 'pylint-report-*-chg.log' ]], failedTotalAll: '0', useDeltaValues: false, @@ -159,31 +112,84 @@ timeout(time: global_timeout, unit: 'HOURS') { ]) } } - } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { - cleanWs notFailBuild: true - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - if (env.NODE_NAME.startsWith('jenkins-pr-')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: 'The lint test passed', - status: 'SUCCESS', - context: "jenkins/pr/lint" + stage('Lint Full') { + if (env.CHANGE_BRANCH =~ /(?i)^merge[._-]/) { + // perform a full linit if this is a merge forward and the change only lint passed. + try { + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: 'Python lint on everything begins...', + status: 'PENDING', + context: "jenkins/pr/lint" + } + parallel( + lintSaltFull: { + stage('Lint Salt Full') { + sh shell_header + ''' + eval "$(pyenv init - --no-rehash)" + pyenv shell 2.7.15 + EC=254 + export PYLINT_REPORT=pylint-report-salt-full.log + nox -e lint-salt + EC=$? + exit $EC + ''' + } + }, + lintTestsFull: { + stage('Lint Tests Full') { + sh shell_header + ''' + eval "$(pyenv init - --no-rehash)" + pyenv shell 2.7.15 + EC=254 + export PYLINT_REPORT=pylint-report-tests-full.log + nox -e lint-salt + EC=$? + exit $EC + ''' + } + } + ) + } finally { + archiveArtifacts artifacts: 'pylint-report-*-full.log', allowEmptyArchive: true + step([$class: 'WarningsPublisher', + parserConfigurations: [[ + parserName: 'PyLint', + pattern: 'pylint-report-*-full.log' + ]], + failedTotalAll: '0', + useDeltaValues: false, + canRunOnFailed: true, + usePreviousBuildAsReference: true + ]) + } + } } - } else { - if (env.NODE_NAME.startsWith('jenkins-pr-')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: 'The lint test failed', - status: 'FAILURE', - context: "jenkins/pr/lint" - } - try { - slackSend channel: "#jenkins-prod-pr", - color: '#FF0000', - message: "FAILED: PR-Job: '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" - } catch (Exception e) { - sh 'echo Failed to send the Slack notification' + } catch (Exception e) { + currentBuild.result = 'FAILURE' + } finally { + cleanWs notFailBuild: true + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: 'The lint test passed', + status: 'SUCCESS', + context: "jenkins/pr/lint" + } + } else { + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: 'The lint test failed', + status: 'FAILURE', + context: "jenkins/pr/lint" + } + try { + slackSend channel: "#jenkins-prod-pr", + color: '#FF0000', + message: "FAILED: PR-Job: '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" + } catch (Exception e) { + sh 'echo Failed to send the Slack notification' + } } } } From d76ef84c0e6b7bd31b9d56bca0778827a87c73af Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 24 Jun 2019 11:38:00 +0100 Subject: [PATCH 31/31] Minor fixes/adjustments to the new CI pipelines --- .ci/docs | 9 +- .ci/kitchen-centos6-py2 | 10 +- .ci/kitchen-centos7-py2 | 10 +- .ci/kitchen-centos7-py3 | 10 +- .ci/kitchen-debian8-py2 | 10 +- .ci/kitchen-debian8-py3 | 10 +- .ci/kitchen-debian9-py2 | 10 +- .ci/kitchen-debian9-py3 | 10 +- .ci/kitchen-fedora29-py2 | 10 +- .ci/kitchen-fedora29-py3 | 10 +- .ci/kitchen-ubuntu1604-py2 | 12 +- .ci/kitchen-ubuntu1604-py3 | 12 +- .ci/kitchen-ubuntu1804-py2 | 10 +- .ci/kitchen-ubuntu1804-py3 | 10 +- .ci/kitchen-windows2016-py2 | 14 +- .ci/kitchen-windows2016-py3 | 13 +- .ci/lint | 254 ++++++++++++++++++------------------ 17 files changed, 167 insertions(+), 257 deletions(-) diff --git a/.ci/docs b/.ci/docs index 9a85c6a770f..3b2b32d2389 100644 --- a/.ci/docs +++ b/.ci/docs @@ -1,14 +1,9 @@ // Define the maximum time, in hours, that a test run should run for def global_timeout = 2 +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], ]) def shell_header @@ -44,7 +39,7 @@ timeout(time: global_timeout, unit: 'HOURS') { pyenv install --skip-existing 3.6.8 pyenv shell 3.6.8 python --version - pip install -U https://github.com/s0undt3ch/nox/archive/hotfix/py2.zip#egg=Nox==2018.10.17 + pip install -U nox-py2 nox --version ''' } diff --git a/.ci/kitchen-centos6-py2 b/.ci/kitchen-centos6-py2 index 7d88c8d52ae..22f670b138c 100644 --- a/.ci/kitchen-centos6-py2 +++ b/.ci/kitchen-centos6-py2 @@ -7,17 +7,11 @@ def global_timeout = testrun_timeout + 1; def distro_name = 'centos' def distro_version = '6' def python_version = 'py2' -def golden_images_branch = '2018.3' def salt_target_branch = '2018.3' +def golden_images_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], parameters([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) @@ -35,10 +29,10 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "SALT_TARGET_BRANCH=${salt_target_branch}", "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', diff --git a/.ci/kitchen-centos7-py2 b/.ci/kitchen-centos7-py2 index e744c216106..9c79a229e6f 100644 --- a/.ci/kitchen-centos7-py2 +++ b/.ci/kitchen-centos7-py2 @@ -7,17 +7,11 @@ def global_timeout = testrun_timeout + 1; def distro_name = 'centos' def distro_version = '7' def python_version = 'py2' -def golden_images_branch = '2018.3' def salt_target_branch = '2018.3' +def golden_images_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], parameters([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) @@ -35,10 +29,10 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "SALT_TARGET_BRANCH=${salt_target_branch}", "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', diff --git a/.ci/kitchen-centos7-py3 b/.ci/kitchen-centos7-py3 index a1ca8e4740d..288d17ac5af 100644 --- a/.ci/kitchen-centos7-py3 +++ b/.ci/kitchen-centos7-py3 @@ -7,17 +7,11 @@ def global_timeout = testrun_timeout + 1; def distro_name = 'centos' def distro_version = '7' def python_version = 'py3' -def golden_images_branch = '2018.3' def salt_target_branch = '2018.3' +def golden_images_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], parameters([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) @@ -35,10 +29,10 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "SALT_TARGET_BRANCH=${salt_target_branch}", "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', diff --git a/.ci/kitchen-debian8-py2 b/.ci/kitchen-debian8-py2 index fc95515acb6..ea276fc4b02 100644 --- a/.ci/kitchen-debian8-py2 +++ b/.ci/kitchen-debian8-py2 @@ -7,17 +7,11 @@ def global_timeout = testrun_timeout + 1; def distro_name = 'debian' def distro_version = '8' def python_version = 'py2' -def golden_images_branch = '2018.3' def salt_target_branch = '2018.3' +def golden_images_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], parameters([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) @@ -35,10 +29,10 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "SALT_TARGET_BRANCH=${salt_target_branch}", "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', diff --git a/.ci/kitchen-debian8-py3 b/.ci/kitchen-debian8-py3 index 465308952bf..388ecd5be30 100644 --- a/.ci/kitchen-debian8-py3 +++ b/.ci/kitchen-debian8-py3 @@ -7,17 +7,11 @@ def global_timeout = testrun_timeout + 1; def distro_name = 'debian' def distro_version = '8' def python_version = 'py3' -def golden_images_branch = '2018.3' def salt_target_branch = '2018.3' +def golden_images_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], parameters([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) @@ -35,10 +29,10 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "SALT_TARGET_BRANCH=${salt_target_branch}", "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', diff --git a/.ci/kitchen-debian9-py2 b/.ci/kitchen-debian9-py2 index 819d75a07bb..53e5e43788b 100644 --- a/.ci/kitchen-debian9-py2 +++ b/.ci/kitchen-debian9-py2 @@ -7,17 +7,11 @@ def global_timeout = testrun_timeout + 1; def distro_name = 'debian' def distro_version = '9' def python_version = 'py2' -def golden_images_branch = '2018.3' def salt_target_branch = '2018.3' +def golden_images_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], parameters([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) @@ -35,10 +29,10 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "SALT_TARGET_BRANCH=${salt_target_branch}", "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', diff --git a/.ci/kitchen-debian9-py3 b/.ci/kitchen-debian9-py3 index 2a125497608..a0bedaead0b 100644 --- a/.ci/kitchen-debian9-py3 +++ b/.ci/kitchen-debian9-py3 @@ -7,17 +7,11 @@ def global_timeout = testrun_timeout + 1; def distro_name = 'debian' def distro_version = '9' def python_version = 'py3' -def golden_images_branch = '2018.3' def salt_target_branch = '2018.3' +def golden_images_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], parameters([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) @@ -35,10 +29,10 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "SALT_TARGET_BRANCH=${salt_target_branch}", "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', diff --git a/.ci/kitchen-fedora29-py2 b/.ci/kitchen-fedora29-py2 index b713814a4f4..9a3584aff35 100644 --- a/.ci/kitchen-fedora29-py2 +++ b/.ci/kitchen-fedora29-py2 @@ -7,17 +7,11 @@ def global_timeout = testrun_timeout + 1; def distro_name = 'fedora' def distro_version = '29' def python_version = 'py2' -def golden_images_branch = '2018.3' def salt_target_branch = '2018.3' +def golden_images_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], parameters([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) @@ -35,10 +29,10 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "SALT_TARGET_BRANCH=${salt_target_branch}", "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', diff --git a/.ci/kitchen-fedora29-py3 b/.ci/kitchen-fedora29-py3 index 0b222535d6e..49babf4385c 100644 --- a/.ci/kitchen-fedora29-py3 +++ b/.ci/kitchen-fedora29-py3 @@ -7,17 +7,11 @@ def global_timeout = testrun_timeout + 1; def distro_name = 'fedora' def distro_version = '29' def python_version = 'py3' -def golden_images_branch = '2018.3' def salt_target_branch = '2018.3' +def golden_images_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], parameters([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) @@ -35,10 +29,10 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "SALT_TARGET_BRANCH=${salt_target_branch}", "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', diff --git a/.ci/kitchen-ubuntu1604-py2 b/.ci/kitchen-ubuntu1604-py2 index 04bd927b2e8..6b987ad8c80 100644 --- a/.ci/kitchen-ubuntu1604-py2 +++ b/.ci/kitchen-ubuntu1604-py2 @@ -1,4 +1,4 @@ -//// Define the maximum time, in hours, that a test run should run for +// Define the maximum time, in hours, that a test run should run for def testrun_timeout = 6 // Now define a global pipeline timeout. This is the test run timeout with one(1) additional // hour to allow for artifacts to be downloaded, if possible. @@ -7,17 +7,11 @@ def global_timeout = testrun_timeout + 1; def distro_name = 'ubuntu' def distro_version = '1604' def python_version = 'py2' -def golden_images_branch = '2018.3' def salt_target_branch = '2018.3' +def golden_images_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], parameters([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) @@ -35,10 +29,10 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "SALT_TARGET_BRANCH=${salt_target_branch}", "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', diff --git a/.ci/kitchen-ubuntu1604-py3 b/.ci/kitchen-ubuntu1604-py3 index 67efac98b70..368172f000b 100644 --- a/.ci/kitchen-ubuntu1604-py3 +++ b/.ci/kitchen-ubuntu1604-py3 @@ -1,4 +1,4 @@ -//// Define the maximum time, in hours, that a test run should run for +// Define the maximum time, in hours, that a test run should run for def testrun_timeout = 6 // Now define a global pipeline timeout. This is the test run timeout with one(1) additional // hour to allow for artifacts to be downloaded, if possible. @@ -7,17 +7,11 @@ def global_timeout = testrun_timeout + 1; def distro_name = 'ubuntu' def distro_version = '1604' def python_version = 'py3' -def golden_images_branch = '2018.3' def salt_target_branch = '2018.3' +def golden_images_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], parameters([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) @@ -35,10 +29,10 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "SALT_TARGET_BRANCH=${salt_target_branch}", "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', diff --git a/.ci/kitchen-ubuntu1804-py2 b/.ci/kitchen-ubuntu1804-py2 index 3bf433abf0b..1808273039d 100644 --- a/.ci/kitchen-ubuntu1804-py2 +++ b/.ci/kitchen-ubuntu1804-py2 @@ -7,17 +7,11 @@ def global_timeout = testrun_timeout + 1; def distro_name = 'ubuntu' def distro_version = '1804' def python_version = 'py2' -def golden_images_branch = '2018.3' def salt_target_branch = '2018.3' +def golden_images_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], parameters([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) @@ -35,10 +29,10 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "SALT_TARGET_BRANCH=${salt_target_branch}", "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', diff --git a/.ci/kitchen-ubuntu1804-py3 b/.ci/kitchen-ubuntu1804-py3 index b39748450af..93671cb224c 100644 --- a/.ci/kitchen-ubuntu1804-py3 +++ b/.ci/kitchen-ubuntu1804-py3 @@ -7,17 +7,11 @@ def global_timeout = testrun_timeout + 1; def distro_name = 'ubuntu' def distro_version = '1804' def python_version = 'py3' -def golden_images_branch = '2018.3' def salt_target_branch = '2018.3' +def golden_images_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], parameters([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) @@ -35,10 +29,10 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "SALT_TARGET_BRANCH=${salt_target_branch}", "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', diff --git a/.ci/kitchen-windows2016-py2 b/.ci/kitchen-windows2016-py2 index 8637b711d44..3e9eb1b710e 100644 --- a/.ci/kitchen-windows2016-py2 +++ b/.ci/kitchen-windows2016-py2 @@ -1,5 +1,5 @@ // Define the maximum time, in hours, that a test run should run for -def testrun_timeout = 6 +def testrun_timeout = 8 // Now define a global pipeline timeout. This is the test run timeout with one(1) additional // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; @@ -7,17 +7,11 @@ def global_timeout = testrun_timeout + 1; def distro_name = 'windows' def distro_version = '2016' def python_version = 'py2' -def golden_images_branch = '2018.3' def salt_target_branch = '2018.3' +def golden_images_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], parameters([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) @@ -35,10 +29,10 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', - 'NOX_PASSTHROUGH_OPTS=--ssh-tests', + "NOX_PASSTHROUGH_OPTS=--unit", + "SALT_TARGET_BRANCH=${salt_target_branch}", "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', diff --git a/.ci/kitchen-windows2016-py3 b/.ci/kitchen-windows2016-py3 index bb5fbb05f97..0c011876455 100644 --- a/.ci/kitchen-windows2016-py3 +++ b/.ci/kitchen-windows2016-py3 @@ -1,5 +1,5 @@ // Define the maximum time, in hours, that a test run should run for -def testrun_timeout = 6 +def testrun_timeout = 8 // Now define a global pipeline timeout. This is the test run timeout with one(1) additional // hour to allow for artifacts to be downloaded, if possible. def global_timeout = testrun_timeout + 1; @@ -7,17 +7,11 @@ def global_timeout = testrun_timeout + 1; def distro_name = 'windows' def distro_version = '2016' def python_version = 'py3' -def golden_images_branch = '2018.3' def salt_target_branch = '2018.3' +def golden_images_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], parameters([ booleanParam(defaultValue: true, description: 'Run full test suite', name: 'runFull') ]) @@ -35,9 +29,10 @@ timeout(time: global_timeout, unit: 'HOURS') { 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/nox-platforms.yml', 'SALT_KITCHEN_VERIFIER=/var/jenkins/workspace/nox-verifier.yml', 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - "SALT_TARGET_BRANCH=${salt_target_branch}", 'NOX_ENV_NAME=runtests-zeromq', 'NOX_ENABLE_FROM_FILENAMES=true', + "NOX_PASSTHROUGH_OPTS=--unit", + "SALT_TARGET_BRANCH=${salt_target_branch}", "GOLDEN_IMAGES_CI_BRANCH=${golden_images_branch}", "CODECOV_FLAGS=${distro_name}${distro_version},${python_version}", 'PATH=~/.rbenv/shims:/usr/local/rbenv/shims:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', diff --git a/.ci/lint b/.ci/lint index bf0ffe083db..a1eb8ef99da 100644 --- a/.ci/lint +++ b/.ci/lint @@ -1,49 +1,42 @@ // Define the maximum time, in hours, that a test run should run for def global_timeout = 3 +def salt_target_branch = '2018.3' properties([ buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')), - [ - $class: 'ScannerJobProperty', doNotScan: false - ], - [ - $class: 'RebuildSettings', autoRebuild: false, rebuildDisabled: false - ], ]) def shell_header -def salt_target_branch = '2018.3' timeout(time: global_timeout, unit: 'HOURS') { node('lint') { ansiColor('xterm') { timestamps { - withEnv([ - "SALT_TARGET_BRANCH=${salt_target_branch}", - ]) { - try { - // Set the GH status even before cloning the repo - if (env.NODE_NAME.startsWith('jenkins-pr-')) { - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: 'Python lint begins...', - status: 'PENDING', - context: "jenkins/pr/lint" - } - shell_header = 'export PYENV_ROOT="/usr/local/pyenv"\nexport PATH="$PYENV_ROOT/bin:$PATH"' - } else { - shell_header = '' + try { + // Set the GH status even before cloning the repo + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: 'Python lint begins...', + status: 'PENDING', + context: "jenkins/pr/lint" } + shell_header = 'export PYENV_ROOT="/usr/local/pyenv"\nexport PATH="$PYENV_ROOT/bin:$PATH"' + } else { + shell_header = '' + } + + withEnv(["SALT_TARGET_BRANCH=${salt_target_branch}"]) { // Checkout the repo stage('checkout-scm') { cleanWs notFailBuild: true checkout scm + sh 'git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH}' } // Setup the kitchen required bundle stage('Setup') { sh shell_header + ''' - git fetch --no-tags https://github.com/saltstack/salt.git +refs/heads/${SALT_TARGET_BRANCH}:refs/remotes/origin/${SALT_TARGET_BRANCH} # Need -M to detect renames otherwise they are reported as Delete and Add, need -C to detect copies, -C includes -M # -M is on by default in git 2.9+ git diff --name-status -l99999 -C "origin/${SALT_TARGET_BRANCH}" > file-list-status.log @@ -56,140 +49,151 @@ timeout(time: global_timeout, unit: 'HOURS') { pyenv install --skip-existing 2.7.15 pyenv shell 2.7.15 python --version - pip install -U https://github.com/s0undt3ch/nox/archive/hotfix/py2.zip#egg=Nox==2018.10.17 + pip install -U nox-py2 nox --version # Create the required virtualenvs in serial nox --install-only -e lint-salt nox --install-only -e lint-tests ''' - archiveArtifacts artifacts: 'file-list-status.log,file-list-changed.log,file-list-deleted.log,file-list-experiment.log' } - stage('Lint Changes') { - try { - parallel( - lintSalt: { - stage('Lint Salt Changes') { - if (readFile('file-list-changed.log') =~ /(?i)(^|\n)(salt\/.*\.py|setup\.py)\n/) { - sh shell_header + ''' - eval "$(pyenv init - --no-rehash)" - pyenv shell 2.7.15 - EC=254 - export PYLINT_REPORT=pylint-report-salt-chg.log - grep -Ei '^salt/.*\\.py$|^setup\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' nox -e lint-salt -- - EC=$? - exit $EC - ''' - } - } - }, - lintTests: { - stage('Lint Test Changes') { - if (readFile('file-list-changed.log') =~ /(?i)(^|\n)tests\/.*\.py\n/) { - sh shell_header + ''' - eval "$(pyenv init - --no-rehash)" - pyenv shell 2.7.15 - EC=254 - export PYLINT_REPORT=pylint-report-tests-chg.log - grep -Ei '^tests/.*\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' nox -e lint-tests -- - EC=$? - exit $EC - ''' - } + archiveArtifacts artifacts: 'file-list-status.log,file-list-changed.log,file-list-deleted.log,file-list-experiment.log' + } + + stage('Lint Changes') { + try { + parallel( + lintSalt: { + stage('Lint Salt Changes') { + if (readFile('file-list-changed.log') =~ /(?i)(^|\n)(salt\/.*\.py|setup\.py)\n/) { + sh shell_header + ''' + eval "$(pyenv init - --no-rehash)" + pyenv shell 2.7.15 + EC=254 + export PYLINT_REPORT=pylint-report-salt-chg.log + grep -Ei '^salt/.*\\.py$|^setup\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' nox -e lint-salt -- + EC=$? + exit $EC + ''' } } - ) - } finally { - archiveArtifacts artifacts: 'pylint-report-*-chg.log', allowEmptyArchive: true + }, + lintTests: { + stage('Lint Test Changes') { + if (readFile('file-list-changed.log') =~ /(?i)(^|\n)tests\/.*\.py\n/) { + sh shell_header + ''' + eval "$(pyenv init - --no-rehash)" + pyenv shell 2.7.15 + EC=254 + export PYLINT_REPORT=pylint-report-tests-chg.log + grep -Ei '^tests/.*\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' nox -e lint-tests -- + EC=$? + exit $EC + ''' + } + } + } + ) + } finally { + def changed_logs_pattern = 'pylint-report-*-chg.log' + archiveArtifacts artifacts: changed_logs_pattern, allowEmptyArchive: true + if (env.NODE_NAME.startsWith('jenkins-pr-')) { step([$class: 'WarningsPublisher', parserConfigurations: [[ parserName: 'PyLint', - pattern: 'pylint-report-*-chg.log' + pattern: changed_logs_pattern ]], failedTotalAll: '0', useDeltaValues: false, canRunOnFailed: true, usePreviousBuildAsReference: true ]) + } else { + recordIssues(enabledForFailure: true, tool: pyLint(pattern: changed_logs_pattern, reportEncoding: 'UTF-8')) } } - stage('Lint Full') { - if (env.CHANGE_BRANCH =~ /(?i)^merge[._-]/) { - // perform a full linit if this is a merge forward and the change only lint passed. - try { - if (env.NODE_NAME.startsWith('jenkins-pr-')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: 'Python lint on everything begins...', - status: 'PENDING', - context: "jenkins/pr/lint" - } - parallel( - lintSaltFull: { - stage('Lint Salt Full') { - sh shell_header + ''' - eval "$(pyenv init - --no-rehash)" - pyenv shell 2.7.15 - EC=254 - export PYLINT_REPORT=pylint-report-salt-full.log - nox -e lint-salt - EC=$? - exit $EC - ''' - } - }, - lintTestsFull: { - stage('Lint Tests Full') { - sh shell_header + ''' - eval "$(pyenv init - --no-rehash)" - pyenv shell 2.7.15 - EC=254 - export PYLINT_REPORT=pylint-report-tests-full.log - nox -e lint-salt - EC=$? - exit $EC - ''' - } + } + stage('Lint Full') { + if (env.CHANGE_BRANCH =~ /(?i)^merge[._-]/) { + // perform a full linit if this is a merge forward and the change only lint passed. + try { + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: 'Python lint on everything begins...', + status: 'PENDING', + context: "jenkins/pr/lint" + } + parallel( + lintSaltFull: { + stage('Lint Salt Full') { + sh shell_header + ''' + eval "$(pyenv init - --no-rehash)" + pyenv shell 2.7.15 + EC=254 + export PYLINT_REPORT=pylint-report-salt-full.log + nox -e lint-salt + EC=$? + exit $EC + ''' } - ) - } finally { - archiveArtifacts artifacts: 'pylint-report-*-full.log', allowEmptyArchive: true + }, + lintTestsFull: { + stage('Lint Tests Full') { + sh shell_header + ''' + eval "$(pyenv init - --no-rehash)" + pyenv shell 2.7.15 + EC=254 + export PYLINT_REPORT=pylint-report-tests-full.log + nox -e lint-salt + EC=$? + exit $EC + ''' + } + } + ) + } finally { + def full_logs_pattern = 'pylint-report-*-full.log' + archiveArtifacts artifacts: full_logs_pattern, allowEmptyArchive: true + if (env.NODE_NAME.startsWith('jenkins-pr-')) { step([$class: 'WarningsPublisher', parserConfigurations: [[ parserName: 'PyLint', - pattern: 'pylint-report-*-full.log' + pattern: full_logs_pattern ]], failedTotalAll: '0', useDeltaValues: false, canRunOnFailed: true, usePreviousBuildAsReference: true ]) + } else { + recordIssues(enabledForFailure: true, tool: pyLint(pattern: full_logs_pattern, reportEncoding: 'UTF-8')) } } } - } catch (Exception e) { - currentBuild.result = 'FAILURE' - } finally { - cleanWs notFailBuild: true - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - if (env.NODE_NAME.startsWith('jenkins-pr-')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: 'The lint test passed', - status: 'SUCCESS', - context: "jenkins/pr/lint" - } - } else { - if (env.NODE_NAME.startsWith('jenkins-pr-')) { - githubNotify credentialsId: 'test-jenkins-credentials', - description: 'The lint test failed', - status: 'FAILURE', - context: "jenkins/pr/lint" - } - try { - slackSend channel: "#jenkins-prod-pr", - color: '#FF0000', - message: "FAILED: PR-Job: '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" - } catch (Exception e) { - sh 'echo Failed to send the Slack notification' - } + } + } catch (Exception e) { + currentBuild.result = 'FAILURE' + } finally { + cleanWs notFailBuild: true + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: 'The lint test passed', + status: 'SUCCESS', + context: "jenkins/pr/lint" + } + } else { + if (env.NODE_NAME.startsWith('jenkins-pr-')) { + githubNotify credentialsId: 'test-jenkins-credentials', + description: 'The lint test failed', + status: 'FAILURE', + context: "jenkins/pr/lint" + } + try { + slackSend channel: "#jenkins-prod-pr", + color: '#FF0000', + message: "FAILED: PR-Job: '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" + } catch (Exception e) { + sh 'echo Failed to send the Slack notification' } } }