saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Update the installer build

Browse source 
Fixes some issues with Py3 not building
Brings the 2 build_env files more in line with each other
Fixes some typos in the build_env scripts
Removes the need for seperate req files for Py2 and Py3
Setup.py uses the windows req file on Windows
Removes the need to install pycrypto and pyyaml in setup.py. They are
now handled in the req.txt
Uses pypi resources instead of downloading .whl files and .exes for
dependencies
Updates nssm version to address the issue in #46192
Removes nssm from buildenv and downloads 32 and 64 bit versions of nssm
in build_pkg.bat
Removes the AppNoConsole=1 setting from the nsis script since it is not
needed with the new version of nssm
This commit is contained in:
twangboy 2018-03-02 13:04:48 -07:00
parent 8c2c4e3316
commit bcf8b19566
No known key found for this signature in database
GPG key ID: 93FF3BDEB278C9EB
9 changed files with 49 additions and 169 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
pkg/windows/build_env_2.ps1
View file

@ -206,7 +206,7 @@ if ( ! [bool]$Env:SALT_PIP_LOCAL_CACHE) {
Start_Process_and_test_exitcode "$($ini['Settings']['Python2Dir'])\python.exe" "-m pip download --dest $Env:SALT_PIP_LOCAL_CACHE -r $($script_path)\req_pip.txt" "pip download"
}
Write-Output " reading from local pip cache $Env:SALT_PIP_LOCAL_CACHE"
Write-Output " If a (new) ressource is missing, please delete all files in this cache, go online and repeat"
Write-Output " If a (new) resource is missing, please delete all files in this cache, go online and repeat"
Start_Process_and_test_exitcode "$($ini['Settings']['Python2Dir'])\python.exe" "-m pip install --no-index --find-links=$Env:SALT_PIP_LOCAL_CACHE -r $($script_path)\req_pip.txt" "pip install"
}
@ -218,21 +218,26 @@ Write-Output " ----------------------------------------------------------------"
Write-Output " - $script_name :: Installing pypi resources using pip . . ."
Write-Output " ----------------------------------------------------------------"
if ( ! [bool]$Env:SALT_REQ_LOCAL_CACHE) {
Start_Process_and_test_exitcode "$($ini['Settings']['Scripts2Dir'])\pip.exe" "--no-cache-dir install -r $($script_path)\req_2.txt" "pip install"
Start_Process_and_test_exitcode "$($ini['Settings']['Scripts2Dir'])\pip.exe" "--no-cache-dir install -r $($script_path)\req.txt" "pip install"
} else {
if ( (Get-ChildItem $Env:SALT_REQ_LOCAL_CACHE | Measure-Object).Count -eq 0 ) {
# folder empty
Write-Output " pip download from req_2.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE"
Start_Process_and_test_exitcode "$($ini['Settings']['Python2Dir'])\python.exe" "-m pip download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_2.txt" "pip download"
Write-Output " pip download from req.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE"
Start_Process_and_test_exitcode "$($ini['Settings']['Python2Dir'])\python.exe" "-m pip download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip download"
}
Write-Output " reading from local pip cache $Env:SALT_REQ_LOCAL_CACHE"
Write-Output " If a (new) ressource is missing, please delete all files in this cache, go online and repeat"
Start_Process_and_test_exitcode "$($ini['Settings']['Python2Dir'])\python.exe" "-m pip install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_2.txt" "pip install"
Write-Output " If a (new) resource is missing, please delete all files in this cache, go online and repeat"
Start_Process_and_test_exitcode "$($ini['Settings']['Python2Dir'])\python.exe" "-m pip install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip install"
}
#==============================================================================
# Move PyWin32 DLL's to site-packages\win32
# Cleaning Up PyWin32
#==============================================================================
Write-Output " ----------------------------------------------------------------"
Write-Output " - $script_name :: Cleaning Up PyWin32 . . ."
Write-Output " ----------------------------------------------------------------"
# Move DLL's to Python Root
Write-Output " - $script_name :: Moving PyWin32 DLLs . . ."
Move-Item "$($ini['Settings']['SitePkgs2Dir'])\pywin32_system32\*.dll" "$($ini['Settings']['SitePkgs2Dir'])\win32" -Force
@ -248,37 +253,6 @@ Remove-Item "$($ini['Settings']['SitePkgs2Dir'])\pythonwin" -Force -Recurse
Write-Output " - $script_name :: Removing PyWin32 scripts . . ."
Remove-Item "$($ini['Settings']['Scripts2Dir'])\pywin32_*" -Force -Recurse
#==============================================================================
# Install PyYAML with CLoader
# This has to be a compiled binary to get the CLoader
#==============================================================================
Write-Output " ----------------------------------------------------------------"
Write-Output " - $script_name :: Installing PyYAML . . ."
Write-Output " ----------------------------------------------------------------"
# Download
$file = "$($ini[$bitPrograms]['PyYAML2'])"
$url = "$($ini['Settings']['SaltRepo'])/$bitFolder/$file"
$file = "$($ini['Settings']['DownloadDir'])\$bitFolder\$file"
DownloadFileWithProgress $url $file
# Install
Start_Process_and_test_exitcode "$($ini['Settings']['Scripts2Dir'])\easy_install.exe" "-Z $file " "easy_install PyYAML"
#==============================================================================
# Install PyCrypto from wheel file
#==============================================================================
Write-Output " ----------------------------------------------------------------"
Write-Output " - $script_name :: Installing PyCrypto . . ."
Write-Output " ----------------------------------------------------------------"
# Download
$file = "$($ini[$bitPrograms]['PyCrypto2'])"
$url = "$($ini['Settings']['SaltRepo'])/$bitFolder/$file"
$file = "$($ini['Settings']['DownloadDir'])\$bitFolder\$file"
DownloadFileWithProgress $url $file
# Install
Start_Process_and_test_exitcode "$($ini['Settings']['Scripts2Dir'])\pip.exe" "install --no-index --find-links=$($ini['Settings']['DownloadDir']) $file " "pip install PyCrypto"
#==============================================================================
# Copy DLLs to Python Directory
#==============================================================================

28
pkg/windows/build_env_3.ps1
View file

@ -191,7 +191,7 @@ If (!($Path.ToLower().Contains("$($ini['Settings']['Scripts3Dir'])".ToLower())))
#==============================================================================
# Update PIP and SetupTools
# caching depends on environmant variable SALT_PIP_LOCAL_CACHE
# caching depends on environment variable SALT_PIP_LOCAL_CACHE
#==============================================================================
Write-Output " ----------------------------------------------------------------"
Write-Output " - $script_name :: Updating PIP and SetupTools . . ."
@ -206,44 +206,36 @@ if ( ! [bool]$Env:SALT_PIP_LOCAL_CACHE) {
Start_Process_and_test_exitcode "$($ini['Settings']['Python3Dir'])\python.exe" "-m pip download --dest $Env:SALT_PIP_LOCAL_CACHE -r $($script_path)\req_pip.txt" "pip download"
}
Write-Output " reading from local pip cache $Env:SALT_PIP_LOCAL_CACHE"
Write-Output " If a (new) ressource is missing, please delete all files in this cache, go online and repeat"
Write-Output " If a (new) resource is missing, please delete all files in this cache, go online and repeat"
Start_Process_and_test_exitcode "$($ini['Settings']['Python3Dir'])\python.exe" "-m pip install --no-index --find-links=$Env:SALT_PIP_LOCAL_CACHE -r $($script_path)\req_pip.txt" "pip install"
}
#==============================================================================
# Install pypi resources using pip
# caching depends on environmant variable SALT_REQ_LOCAL_CACHE
# caching depends on environment variable SALT_REQ_LOCAL_CACHE
#==============================================================================
Write-Output " ----------------------------------------------------------------"
Write-Output " - $script_name :: Installing pypi resources using pip . . ."
Write-Output " ----------------------------------------------------------------"
if ( ! [bool]$Env:SALT_REQ_LOCAL_CACHE) {
Start_Process_and_test_exitcode "$($ini['Settings']['Scripts3Dir'])\pip.exe" "--no-cache-dir install -r $($script_path)\req_3.txt" "pip install"
Start_Process_and_test_exitcode "$($ini['Settings']['Scripts3Dir'])\pip.exe" "--no-cache-dir install -r $($script_path)\req.txt" "pip install"
} else {
if ( (Get-ChildItem $Env:SALT_REQ_LOCAL_CACHE | Measure-Object).Count -eq 0 ) {
# folder empty
Write-Output " pip download from req_3.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE"
Start_Process_and_test_exitcode "$($ini['Settings']['Python3Dir'])\python.exe" "-m pip download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_3.txt" "pip download"
Write-Output " pip download from req.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE"
Start_Process_and_test_exitcode "$($ini['Settings']['Python3Dir'])\python.exe" "-m pip download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip download"
}
Write-Output " reading from local pip cache $Env:SALT_REQ_LOCAL_CACHE"
Write-Output " If a (new) ressource is missing, please delete all files in this cache, go online and repeat"
Start_Process_and_test_exitcode "$($ini['Settings']['Python3Dir'])\python.exe" "-m pip install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_3.txt" "pip install"
Write-Output " If a (new) resource is missing, please delete all files in this cache, go online and repeat"
Start_Process_and_test_exitcode "$($ini['Settings']['Python3Dir'])\python.exe" "-m pip install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip install"
}
#==============================================================================
# Install PyWin32 from wheel file
# Cleaning Up PyWin32
#==============================================================================
Write-Output " ----------------------------------------------------------------"
Write-Output " - $script_name :: Installing PyWin32 . . ."
Write-Output " - $script_name :: Cleaning Up PyWin32 . . ."
Write-Output " ----------------------------------------------------------------"
# Download
$file = "$($ini[$bitPrograms]['PyWin323'])"
$url = "$($ini['Settings']['SaltRepo'])/$bitFolder/$file"
$file = "$($ini['Settings']['DownloadDir'])\$bitFolder\$file"
DownloadFileWithProgress $url $file
# Install
Start_Process_and_test_exitcode "$($ini['Settings']['Scripts3Dir'])\pip.exe" "install --no-index --find-links=$($ini['Settings']['DownloadDir']) $file " "pip install PyWin32"
# Move DLL's to Python Root
Write-Output " - $script_name :: Moving PyWin32 DLLs . . ."

17
pkg/windows/build_pkg.bat
View file

@ -113,6 +113,23 @@ xcopy /Q /Y "%SrcDir%\conf\master" "%CnfDir%\"
xcopy /Q /Y "%SrcDir%\conf\minion" "%CnfDir%\"
@echo.
@echo Copying NSSM to buildenv
@echo ----------------------------------------------------------------------
:: Make sure the "prereq" directory exists
If NOT Exist "%PreDir%" mkdir "%PreDir%"
:: Set the location of the nssm to download
Set Url64="https://repo.saltstack.com/windows/dependencies/64/nssm-2.24-101-g897c7ad.exe"
Set Url32="https://repo.saltstack.com/windows/dependencies/32/nssm-2.24-101-g897c7ad.exe"
:: Check for 64 bit by finding the Program Files (x86) directory
If Defined ProgramFiles(x86) (
powershell -ExecutionPolicy RemoteSigned -File download_url_file.ps1 -url "%Url64%" -file "%BldDir%\nssm.exe"
) Else (
powershell -ExecutionPolicy RemoteSigned -File download_url_file.ps1 -url "%Url32%" -file "%BldDir%\nssm.exe"
)
@echo.
@echo Copying VCRedist to Prerequisites
@echo ----------------------------------------------------------------------
:: Make sure the "prereq" directory exists

BIN
pkg/windows/buildenv/nssm.exe
View file

Binary file not shown.

1
pkg/windows/installer/Salt-Minion-Setup.nsi
View file

@ -403,7 +403,6 @@ Section -Post
nsExec::Exec "nssm.exe install salt-minion $INSTDIR\bin\python.exe -E -s $INSTDIR\bin\Scripts\salt-minion -c $INSTDIR\conf -l quiet"
nsExec::Exec "nssm.exe set salt-minion Description Salt Minion from saltstack.com"
nsExec::Exec "nssm.exe set salt-minion Start SERVICE_AUTO_START"
nsExec::Exec "nssm.exe set salt-minion AppNoConsole 1"
nsExec::Exec "nssm.exe set salt-minion AppStopMethodConsole 24000"
nsExec::Exec "nssm.exe set salt-minion AppStopMethodWindow 2000"

4
pkg/windows/req_base.txt → pkg/windows/req.txt
View file

@ -13,17 +13,21 @@ ioloop==0.1a0
ipaddress==1.0.18
Jinja2==2.9.6
libnacl==1.6.1
lxml==3.7.3
Mako==1.0.6
MarkupSafe==1.0
msgpack-python==0.4.8
psutil==5.2.2
pyasn1==0.2.3
pycparser==2.17
pycrypto==2.6.1
pycurl==7.43.0
PyMySQL==0.7.11
pyOpenSSL==17.0.0
python-dateutil==2.6.0
python-gnupg==0.4.0
pywin32==223
PyYAML==3.12
pyzmq==16.0.2
requests==2.13.0
singledispatch==3.4.0.3

4
pkg/windows/req_2.txt
View file

@ -1,4 +0,0 @@
-r req_base.txt
lxml==3.6.0
pypiwin32==219

5
pkg/windows/req_3.txt
View file

@ -1,5 +0,0 @@
-r req_base.txt
lxml==3.7.3
pycrypto==2.6.1
PyYAML==3.12

109
setup.py
View file

@ -122,12 +122,7 @@ SALT_SYSPATHS_HARDCODED = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', '
SALT_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'base.txt')
SALT_ZEROMQ_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'zeromq.txt')
SALT_RAET_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'raet.txt')
if IS_PY3:
SALT_WINDOWS_REQS = os.path.join(
os.path.abspath(SETUP_DIRNAME), 'pkg', 'windows', 'req_3.txt')
else:
SALT_WINDOWS_REQS = os.path.join(
os.path.abspath(SETUP_DIRNAME), 'pkg', 'windows', 'req_2.txt')
SALT_WINDOWS_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'pkg', 'windows', 'req.txt')
# Salt SSH Packaging Detection
PACKAGED_FOR_SALT_SSH_FILE = os.path.join(os.path.abspath(SETUP_DIRNAME), '.salt-ssh-package')
@ -150,9 +145,6 @@ def _parse_requirements_file(requirements_file):
if IS_WINDOWS_PLATFORM:
if 'libcloud' in line:
continue
if 'pycrypto' in line.lower() and not IS_PY3:
# On Python 2 in Windows we install PyCrypto using python wheels
continue
if 'm2crypto' in line.lower() and __saltstack_version__.info < (2015, 8): # pylint: disable=undefined-variable
# In Windows, we're installing M2CryptoWin{32,64} which comes
# compiled
@ -325,18 +317,6 @@ if WITH_SETUPTOOLS:
self.run_command('install-m2crypto-windows')
self.distribution.salt_installing_m2crypto_windows = None
if not IS_PY3:
# Install PyCrypto
self.distribution.salt_installing_pycrypto_windows = True
self.run_command('install-pycrypto-windows')
self.distribution.salt_installing_pycrypto_windows = None
# Install PyYAML
self.distribution.salt_installing_pyyaml_windows = True
self.run_command('install-pyyaml-windows')
self.distribution.salt_installing_pyyaml_windows = None
# Download the required DLLs
self.distribution.salt_download_windows_dlls = True
self.run_command('download-windows-dlls')
@ -379,37 +359,6 @@ class InstallM2CryptoWindows(Command):
)
class InstallPyCryptoWindowsWheel(Command):
description = 'Install PyCrypto on Windows'
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self):
if getattr(self.distribution, 'salt_installing_pycrypto_windows', None) is None:
print('This command is not meant to be called on it\'s own')
exit(1)
import platform
from pip.utils import call_subprocess
from pip.utils.logging import indent_log
platform_bits, _ = platform.architecture()
call_arguments = ['pip', 'install', 'wheel']
if platform_bits == '64bit':
call_arguments.append(
'https://repo.saltstack.com/windows/dependencies/64/pycrypto-2.6.1-cp27-none-win_amd64.whl'
)
else:
call_arguments.append(
'https://repo.saltstack.com/windows/dependencies/32/pycrypto-2.6.1-cp27-none-win32.whl'
)
with indent_log():
call_subprocess(call_arguments)
def uri_to_resource(resource_file):
# ## Returns the URI for a resource
# The basic case is that the resource is on saltstack.com
@ -432,37 +381,6 @@ def uri_to_resource(resource_file):
return cached_resource
class InstallCompiledPyYaml(Command):
description = 'Install PyYAML on Windows'
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self):
if getattr(self.distribution, 'salt_installing_pyyaml_windows', None) is None:
print('This command is not meant to be called on it\'s own')
exit(1)
import platform
from pip.utils import call_subprocess
from pip.utils.logging import indent_log
platform_bits, _ = platform.architecture()
call_arguments = ['easy_install', '-Z']
if platform_bits == '64bit':
call_arguments.append(
uri_to_resource('64/PyYAML-3.11.win-amd64-py2.7.exe')
)
else:
call_arguments.append(
uri_to_resource('32/PyYAML-3.11.win32-py2.7.exe')
)
with indent_log():
call_subprocess(call_arguments)
class DownloadWindowsDlls(Command):
description = 'Download required DLL\'s for windows'
@ -779,17 +697,6 @@ class Install(install):
self.run_command('install-m2crypto-windows')
self.distribution.salt_installing_m2crypto_windows = None
if not IS_PY3:
# Install PyCrypto
self.distribution.salt_installing_pycrypto_windows = True
self.run_command('install-pycrypto-windows')
self.distribution.salt_installing_pycrypto_windows = None
# Install PyYAML
self.distribution.salt_installing_pyyaml_windows = True
self.run_command('install-pyyaml-windows')
self.distribution.salt_installing_pyyaml_windows = None
# Download the required DLLs
self.distribution.salt_download_windows_dlls = True
self.run_command('download-windows-dlls')
@ -928,12 +835,7 @@ class SaltDistribution(distutils.dist.Distribution):
self.cmdclass.update({'sdist': CloudSdist,
'install_lib': InstallLib})
if IS_WINDOWS_PLATFORM:
if IS_PY3:
self.cmdclass.update({'download-windows-dlls': DownloadWindowsDlls})
else:
self.cmdclass.update({'install-pycrypto-windows': InstallPyCryptoWindowsWheel,
'install-pyyaml-windows': InstallCompiledPyYaml,
'download-windows-dlls': DownloadWindowsDlls})
self.cmdclass.update({'download-windows-dlls': DownloadWindowsDlls})
if __saltstack_version__.info < (2015, 8): # pylint: disable=undefined-variable
self.cmdclass.update({'install-m2crypto-windows': InstallM2CryptoWindows})
@ -1059,13 +961,14 @@ class SaltDistribution(distutils.dist.Distribution):
def _property_install_requires(self):
install_requires = _parse_requirements_file(SALT_REQS)
if IS_WINDOWS_PLATFORM:
install_requires += _parse_requirements_file(SALT_WINDOWS_REQS)
if self.salt_transport == 'zeromq':
install_requires += _parse_requirements_file(SALT_ZEROMQ_REQS)
elif self.salt_transport == 'raet':
install_requires += _parse_requirements_file(SALT_RAET_REQS)
if IS_WINDOWS_PLATFORM:
install_requires = _parse_requirements_file(SALT_WINDOWS_REQS)
return install_requires
@property