Merge pull request #51976 from s0undt3ch/features/nox-2019.2

[2019.2] Nox fixes

Gemfile

@@ -4,7 +4,7 @@ source 'https://rubygems.org'
 
 # Point this back at the test-kitchen package after 1.23.3 is relased
 gem 'test-kitchen', '~>1.23.3'
-gem 'kitchen-salt', '~>0.4.1'
+gem 'kitchen-salt', :git => 'https://github.com/s0undt3ch/kitchen-salt.git', :branch => 'features/nox'
 gem 'kitchen-sync'
 gem 'git'
 
@@ -14,7 +14,8 @@ end
 
 group :windows do
   gem 'winrm', '~>2.0'
-  gem 'winrm-fs', '~>1.3.1'
+#  gem 'winrm-fs', '~>1.3.1'
+  gem 'winrm-fs', :git => 'https://github.com/s0undt3ch/winrm-fs.git', :branch => 'hotfix/saltstack-ci'
 end
 
 group :ec2 do

noxfile.py

@@ -23,6 +23,10 @@ import nox
 REPO_ROOT = os.path.abspath(os.path.dirname(__file__))
 SITECUSTOMIZE_DIR = os.path.join(REPO_ROOT, 'tests', 'support', 'coverage')
 
+# We can't just import salt because if this is running under a frozen nox, there
+# will be no salt to import
+IS_WINDOWS = sys.platform.lower().startswith('win')
+
 # Python versions to run against
 _PYTHON_VERSIONS = ('2', '2.7', '3', '3.4', '3.5', '3.6')
 
@@ -41,7 +45,9 @@ def _create_ci_directories():
 
 def _install_requirements(session, *extra_requirements):
     # Install requirements
-    _requirements_files = []
+    _requirements_files = [
+        os.path.join(REPO_ROOT, 'requirements', 'pytest.txt')
+    ]
     if sys.platform.startswith('linux'):
         requirements_files = [
             os.path.join(REPO_ROOT, 'requirements', 'tests.txt')
@@ -49,7 +55,6 @@ def _install_requirements(session, *extra_requirements):
     elif sys.platform.startswith('win'):
         requirements_files = [
             os.path.join(REPO_ROOT, 'pkg', 'windows', 'req.txt'),
-            os.path.join(REPO_ROOT, 'pkg', 'windows', 'req_testing.txt'),
         ]
     elif sys.platform.startswith('darwin'):
         requirements_files = [
@@ -61,6 +66,10 @@ def _install_requirements(session, *extra_requirements):
         if not requirements_files:
             break
         requirements_file = requirements_files.pop(0)
+
+        if requirements_file not in _requirements_files:
+            _requirements_files.append(requirements_file)
+
         session.log('Processing {}'.format(requirements_file))
         with open(requirements_file) as rfh:  # pylint: disable=resource-leakage
             for line in rfh:
@@ -80,8 +89,14 @@ def _install_requirements(session, *extra_requirements):
     if extra_requirements:
         session.install(*extra_requirements)
 
+    if IS_WINDOWS:
+        # Windows hacks :/
+        nox_windows_setup = os.path.join(REPO_ROOT, 'tests', 'support', 'nox-windows-setup.py')
+        session.run('python', nox_windows_setup)
+
 
 def _run_with_coverage(session, *test_cmd):
+    session.install('coverage')
     session.run('coverage', 'erase')
     python_path_env_var = os.environ.get('PYTHONPATH') or None
     if python_path_env_var is None:

tests/support/nox-windows-setup.py

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+'''
+    tests.support.nox-windows-setup
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    This script is meant to run under the nox virtualenv to take care of required
+    windows procedures
+'''
+# pylint: disable=resource-leakage
+
+from __future__ import absolute_import, print_function, unicode_literals
+import os
+import re
+import sys
+import site
+import shutil
+
+try:
+    import site
+    SITE_PACKAGES = site.getsitepackages()
+    PYTHON_EXECUTABLE_DIRECTORY = os.path.dirname(sys.executable)
+    PYTHON_SCRIPTS_DIR = os.path.join(PYTHON_EXECUTABLE_DIRECTORY, 'Scripts')
+except AttributeError:
+    # The site module does not have the getsitepackages function when running within a virtualenv
+    # But the site-packages directory WILL be on sys.path
+    SITE_PACKAGES = None
+    for entry in sys.path:
+        if 'site-packages' in entry:
+            SITE_PACKAGES = entry
+            break
+    # Under a virtualenv, the python "binary" is under Scripts already.
+    # Well, not the binary, but the Python DLLs
+    PYTHON_EXECUTABLE_DIRECTORY = PYTHON_SCRIPTS_DIR = os.path.dirname(sys.executable)
+
+# Requests is a Salt dependency, it's safe to import, but...
+try:
+    import requests
+    HAS_REQUESTS = True
+except ImportError:
+    HAS_REQUESTS = False
+
+IS_64_BITS = sys.maxsize > 2**32
+SALT_REPO_URL = 'https://repo.saltstack.com/windows/dependencies/{}'.format(IS_64_BITS and 64 or 32)
+DLLS = ("libeay32.dll", "ssleay32.dll", "OpenSSL_License.txt", "msvcr120.dll", "libsodium.dll")
+
+for dll in DLLS:
+    outfile = os.path.join(PYTHON_EXECUTABLE_DIRECTORY, dll)
+    if os.path.exists(outfile):
+        continue
+    src_url = '{}/{}'.format(SALT_REPO_URL, dll)
+    if HAS_REQUESTS:
+        print('Downloading {} to {}'.format(src_url, outfile))
+        request = requests.get(src_url, allow_redirects=True)
+        with open(outfile, 'wb') as wfh:
+            wfh.write(request.content)
+    else:
+        print('ATTENTION: The python requests package is not installed, can\'t download {}'.format(src_url))
+
+PYWIN32_SYSTEM32_DIR = os.path.join(SITE_PACKAGES, 'pywin32_system32')
+if os.path.exists(PYWIN32_SYSTEM32_DIR):
+    for fname in os.listdir(PYWIN32_SYSTEM32_DIR):
+        if not fname.endswith('.dll'):
+            continue
+        spath = os.path.join(PYWIN32_SYSTEM32_DIR, fname)
+        dpath = spath.replace('pywin32_system32', 'win32')
+        print('Moving {} to {}'.format(spath, dpath))
+        shutil.move(spath, dpath)
+
+    print('Deleting {}'.format(PYWIN32_SYSTEM32_DIR))
+    shutil.rmtree(PYWIN32_SYSTEM32_DIR, ignore_errors=True)
+
+
+if os.path.exists(PYTHON_SCRIPTS_DIR):
+    print('Searching for pywin32 scripts to delete')
+    for fname in os.listdir(PYTHON_SCRIPTS_DIR):
+        if not fname.startswith('pywin32_'):
+            continue
+        fpath = os.path.join(PYTHON_SCRIPTS_DIR, fname)
+        print('Deleting {}'.format(fpath))
+        os.unlink(fpath)
+
+
+PYTHONWIN_DIR = os.path.join(SITE_PACKAGES, 'pythonwin')
+if os.path.exists(PYTHONWIN_DIR):
+    print('Deleting {}'.format(PYTHONWIN_DIR))
+    shutil.rmtree(PYTHONWIN_DIR, ignore_errors=True)
+
+PYCRPTO_NT_FILE = os.path.join(SITE_PACKAGES, 'Crypto', 'Random', 'OSRNG', 'nt.py')
+if os.path.exists(PYCRPTO_NT_FILE):
+    with open(PYCRPTO_NT_FILE, 'r') as rfh:
+        contents = rfh.read()
+        new_contents = re.sub(
+            r'^import winrandom$',
+            'from Crypto.Random.OSRNG import winrandom',
+            contents,
+            count=1,
+            flags=re.MULTILINE
+        )
+        if contents != new_contents:
+            print('Patching {}'.format(PYCRPTO_NT_FILE))
+            with open(PYCRPTO_NT_FILE, 'w') as wfh:
+                wfh.write(new_contents)