diff --git a/conf/minion b/conf/minion index 6cae0432950..fa5caf317b9 100644 --- a/conf/minion +++ b/conf/minion @@ -373,7 +373,7 @@ # interface: eth0 # cidr: '10.0.0.0/8' -# The number of seconds a mine update runs. +# The number of minutes between mine updates. #mine_interval: 60 # Windows platforms lack posix IPC and must rely on slower TCP based inter- diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index 797aa214f76..e5779d7584c 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -706,7 +706,7 @@ Note these can be defined in the pillar for a minion as well. Default: ``60`` -The number of seconds a mine update runs. +The number of minutes between mine updates. .. code-block:: yaml diff --git a/doc/ref/configuration/proxy.rst b/doc/ref/configuration/proxy.rst index 974e0af8902..e55f3fc01b5 100644 --- a/doc/ref/configuration/proxy.rst +++ b/doc/ref/configuration/proxy.rst @@ -118,3 +118,53 @@ has to be closed after every command. .. code-block:: yaml proxy_always_alive: False + +``proxy_merge_pillar_in_opts`` +------------------------------ + +.. versionadded:: 2017.7.3 + +Default: ``False``. + +Wheter the pillar data to be merged into the proxy configuration options. +As multiple proxies can run on the same server, we may need different +configuration options for each, while there's one single configuration file. +The solution is merging the pillar data of each proxy minion into the opts. + +.. code-block:: yaml + + proxy_merge_pillar_in_opts: True + +``proxy_deep_merge_pillar_in_opts`` +----------------------------------- + +.. versionadded:: 2017.7.3 + +Default: ``False``. + +Deep merge of pillar data into configuration opts. +This option is evaluated only when :conf_proxy:`proxy_merge_pillar_in_opts` is +enabled. + +``proxy_merge_pillar_in_opts_strategy`` +--------------------------------------- + +.. versionadded:: 2017.7.3 + +Default: ``smart``. + +The strategy used when merging pillar configuration into opts. +This option is evaluated only when :conf_proxy:`proxy_merge_pillar_in_opts` is +enabled. + +``proxy_mines_pillar`` +---------------------- + +.. versionadded:: 2017.7.3 + +Default: ``True``. + +Allow enabling mine details using pillar data. This evaluates the mine +configuration under the pillar, for the following regular minion options that +are also equally available on the proxy minion: :conf_minion:`mine_interval`, +and :conf_minion:`mine_functions`. diff --git a/doc/topics/windows/windows-package-manager.rst b/doc/topics/windows/windows-package-manager.rst index 9d1838c807d..20ed60baf68 100644 --- a/doc/topics/windows/windows-package-manager.rst +++ b/doc/topics/windows/windows-package-manager.rst @@ -481,11 +481,17 @@ Alternatively the ``uninstaller`` can also simply repeat the URL of the msi file :param bool allusers: This parameter is specific to `.msi` installations. It tells `msiexec` to install the software for all users. The default is True. -:param bool cache_dir: If true, the entire directory where the installer resides - will be recursively cached. This is useful for installers that depend on - other files in the same directory for installation. +:param bool cache_dir: If true when installer URL begins with salt://, the + entire directory where the installer resides will be recursively cached. + This is useful for installers that depend on other files in the same + directory for installation. -.. note:: Only applies to salt: installer URLs. +:param str cache_file: + When installer URL begins with salt://, this indicates single file to copy + down for use with the installer. Copied to the same location as the + installer. Use this over ``cache_dir`` if there are many files in the + directory and you only need a specific file and don't want to cache + additional files that may reside in the installer directory. Here's an example for a software package that has dependent files: diff --git a/pkg/windows/installer/Salt-Minion-Setup.nsi b/pkg/windows/installer/Salt-Minion-Setup.nsi index 46fb821fb8b..a8efca21018 100644 --- a/pkg/windows/installer/Salt-Minion-Setup.nsi +++ b/pkg/windows/installer/Salt-Minion-Setup.nsi @@ -44,7 +44,7 @@ ${StrStrAdv} !define CPUARCH "x86" !endif -; Part of the Trim function for Strings +# Part of the Trim function for Strings !define Trim "!insertmacro Trim" !macro Trim ResultVar String Push "${String}" @@ -61,27 +61,27 @@ ${StrStrAdv} !define MUI_UNICON "salt.ico" !define MUI_WELCOMEFINISHPAGE_BITMAP "panel.bmp" -; Welcome page +# Welcome page !insertmacro MUI_PAGE_WELCOME -; License page +# License page !insertmacro MUI_PAGE_LICENSE "LICENSE.txt" -; Configure Minion page +# Configure Minion page Page custom pageMinionConfig pageMinionConfig_Leave -; Instfiles page +# Instfiles page !insertmacro MUI_PAGE_INSTFILES -; Finish page (Customized) +# Finish page (Customized) !define MUI_PAGE_CUSTOMFUNCTION_SHOW pageFinish_Show !define MUI_PAGE_CUSTOMFUNCTION_LEAVE pageFinish_Leave !insertmacro MUI_PAGE_FINISH -; Uninstaller pages +# Uninstaller pages !insertmacro MUI_UNPAGE_INSTFILES -; Language files +# Language files !insertmacro MUI_LANGUAGE "English" @@ -201,8 +201,8 @@ ShowInstDetails show ShowUnInstDetails show -; Check and install Visual C++ redist packages -; See http://blogs.msdn.com/b/astebner/archive/2009/01/29/9384143.aspx for more info +# Check and install Visual C++ redist packages +# See http://blogs.msdn.com/b/astebner/archive/2009/01/29/9384143.aspx for more info Section -Prerequisites Var /GLOBAL VcRedistName @@ -211,12 +211,12 @@ Section -Prerequisites Var /Global CheckVcRedist StrCpy $CheckVcRedist "False" - ; Visual C++ 2015 redist packages + # Visual C++ 2015 redist packages !define PY3_VC_REDIST_NAME "VC_Redist_2015" !define PY3_VC_REDIST_X64_GUID "{50A2BC33-C9CD-3BF1-A8FF-53C10A0B183C}" !define PY3_VC_REDIST_X86_GUID "{BBF2AC74-720C-3CB3-8291-5E34039232FA}" - ; Visual C++ 2008 SP1 MFC Security Update redist packages + # Visual C++ 2008 SP1 MFC Security Update redist packages !define PY2_VC_REDIST_NAME "VC_Redist_2008_SP1_MFC" !define PY2_VC_REDIST_X64_GUID "{5FCE6D76-F5DC-37AB-B2B8-22AB8CEDB1D4}" !define PY2_VC_REDIST_X86_GUID "{9BE518E6-ECC6-35A9-88E4-87755C07200F}" @@ -239,7 +239,7 @@ Section -Prerequisites StrCpy $VcRedistGuid ${PY2_VC_REDIST_X86_GUID} ${EndIf} - ; VCRedist 2008 only needed on Windows Server 2008R2/Windows 7 and below + # VCRedist 2008 only needed on Windows Server 2008R2/Windows 7 and below ${If} ${AtMostWin2008R2} StrCpy $CheckVcRedist "True" ${EndIf} @@ -255,20 +255,41 @@ Section -Prerequisites "$VcRedistName is currently not installed. Would you like to install?" \ /SD IDYES IDNO endVcRedist - ClearErrors - ; The Correct version of VCRedist is copied over by "build_pkg.bat" + # The Correct version of VCRedist is copied over by "build_pkg.bat" SetOutPath "$INSTDIR\" File "..\prereqs\vcredist.exe" - ; /passive used by 2015 installer - ; /qb! used by 2008 installer - ; It just ignores the unrecognized switches... - ExecWait "$INSTDIR\vcredist.exe /qb! /passive" - IfErrors 0 endVcRedist + # If an output variable is specified ($0 in the case below), + # ExecWait sets the variable with the exit code (and only sets the + # error flag if an error occurs; if an error occurs, the contents + # of the user variable are undefined). + # http://nsis.sourceforge.net/Reference/ExecWait + # /passive used by 2015 installer + # /qb! used by 2008 installer + # It just ignores the unrecognized switches... + ClearErrors + ExecWait '"$INSTDIR\vcredist.exe" /qb! /passive /norestart' $0 + IfErrors 0 CheckVcRedistErrorCode MessageBox MB_OK \ "$VcRedistName failed to install. Try installing the package manually." \ /SD IDOK + Goto endVcRedist + + CheckVcRedistErrorCode: + # Check for Reboot Error Code (3010) + ${If} $0 == 3010 + MessageBox MB_OK \ + "$VcRedistName installed but requires a restart to complete." \ + /SD IDOK + + # Check for any other errors + ${ElseIfNot} $0 == 0 + MessageBox MB_OK \ + "$VcRedistName failed with ErrorCode: $0. Try installing the package manually." \ + /SD IDOK + ${EndIf} endVcRedist: + ${EndIf} ${EndIf} @@ -294,12 +315,12 @@ Function .onInit Call parseCommandLineSwitches - ; Check for existing installation + # Check for existing installation ReadRegStr $R0 HKLM \ "Software\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCT_NAME}" \ "UninstallString" StrCmp $R0 "" checkOther - ; Found existing installation, prompt to uninstall + # Found existing installation, prompt to uninstall MessageBox MB_OKCANCEL|MB_ICONEXCLAMATION \ "${PRODUCT_NAME} is already installed.$\n$\n\ Click `OK` to remove the existing installation." \ @@ -307,12 +328,12 @@ Function .onInit Abort checkOther: - ; Check for existing installation of full salt + # Check for existing installation of full salt ReadRegStr $R0 HKLM \ "Software\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCT_NAME_OTHER}" \ "UninstallString" StrCmp $R0 "" skipUninstall - ; Found existing installation, prompt to uninstall + # Found existing installation, prompt to uninstall MessageBox MB_OKCANCEL|MB_ICONEXCLAMATION \ "${PRODUCT_NAME_OTHER} is already installed.$\n$\n\ Click `OK` to remove the existing installation." \ @@ -321,22 +342,22 @@ Function .onInit uninst: - ; Get current Silent status + # Get current Silent status StrCpy $R0 0 ${If} ${Silent} StrCpy $R0 1 ${EndIf} - ; Turn on Silent mode + # Turn on Silent mode SetSilent silent - ; Don't remove all directories + # Don't remove all directories StrCpy $DeleteInstallDir 0 - ; Uninstall silently + # Uninstall silently Call uninstallSalt - ; Set it back to Normal mode, if that's what it was before + # Set it back to Normal mode, if that's what it was before ${If} $R0 == 0 SetSilent normal ${EndIf} @@ -350,7 +371,7 @@ Section -Post WriteUninstaller "$INSTDIR\uninst.exe" - ; Uninstall Registry Entries + # Uninstall Registry Entries WriteRegStr ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_UNINST_KEY}" \ "DisplayName" "$(^Name)" WriteRegStr ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_UNINST_KEY}" \ @@ -366,19 +387,19 @@ Section -Post WriteRegStr HKLM "SYSTEM\CurrentControlSet\services\salt-minion" \ "DependOnService" "nsi" - ; Set the estimated size + # Set the estimated size ${GetSize} "$INSTDIR\bin" "/S=OK" $0 $1 $2 IntFmt $0 "0x%08X" $0 WriteRegDWORD ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_UNINST_KEY}" \ "EstimatedSize" "$0" - ; Commandline Registry Entries + # Commandline Registry Entries WriteRegStr HKLM "${PRODUCT_CALL_REGKEY}" "" "$INSTDIR\salt-call.bat" WriteRegStr HKLM "${PRODUCT_CALL_REGKEY}" "Path" "$INSTDIR\bin\" WriteRegStr HKLM "${PRODUCT_MINION_REGKEY}" "" "$INSTDIR\salt-minion.bat" WriteRegStr HKLM "${PRODUCT_MINION_REGKEY}" "Path" "$INSTDIR\bin\" - ; Register the Salt-Minion Service + # Register the Salt-Minion Service nsExec::Exec "nssm.exe install salt-minion $INSTDIR\bin\python.exe -E -s $INSTDIR\bin\Scripts\salt-minion -c $INSTDIR\conf -l quiet" nsExec::Exec "nssm.exe set salt-minion Description Salt Minion from saltstack.com" nsExec::Exec "nssm.exe set salt-minion Start SERVICE_AUTO_START" @@ -398,12 +419,12 @@ SectionEnd Function .onInstSuccess - ; If StartMinionDelayed is 1, then set the service to start delayed + # If StartMinionDelayed is 1, then set the service to start delayed ${If} $StartMinionDelayed == 1 nsExec::Exec "nssm.exe set salt-minion Start SERVICE_DELAYED_AUTO_START" ${EndIf} - ; If start-minion is 1, then start the service + # If start-minion is 1, then start the service ${If} $StartMinion == 1 nsExec::Exec 'net start salt-minion' ${EndIf} @@ -413,10 +434,11 @@ FunctionEnd Function un.onInit - ; Load the parameters + # Load the parameters ${GetParameters} $R0 # Uninstaller: Remove Installation Directory + ClearErrors ${GetOptions} $R0 "/delete-install-dir" $R1 IfErrors delete_install_dir_not_found StrCpy $DeleteInstallDir 1 @@ -434,7 +456,7 @@ Section Uninstall Call un.uninstallSalt - ; Remove C:\salt from the Path + # Remove C:\salt from the Path Push "C:\salt" Call un.RemoveFromPath @@ -444,27 +466,27 @@ SectionEnd !macro uninstallSalt un Function ${un}uninstallSalt - ; Make sure we're in the right directory + # Make sure we're in the right directory ${If} $INSTDIR == "c:\salt\bin\Scripts" StrCpy $INSTDIR "C:\salt" ${EndIf} - ; Stop and Remove salt-minion service + # Stop and Remove salt-minion service nsExec::Exec 'net stop salt-minion' nsExec::Exec 'sc delete salt-minion' - ; Stop and remove the salt-master service + # Stop and remove the salt-master service nsExec::Exec 'net stop salt-master' nsExec::Exec 'sc delete salt-master' - ; Remove files + # Remove files Delete "$INSTDIR\uninst.exe" Delete "$INSTDIR\nssm.exe" Delete "$INSTDIR\salt*" Delete "$INSTDIR\vcredist.exe" RMDir /r "$INSTDIR\bin" - ; Remove Registry entries + # Remove Registry entries DeleteRegKey ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_UNINST_KEY}" DeleteRegKey ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_UNINST_KEY_OTHER}" DeleteRegKey ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_CALL_REGKEY}" @@ -474,17 +496,17 @@ Function ${un}uninstallSalt DeleteRegKey ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_MINION_REGKEY}" DeleteRegKey ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_RUN_REGKEY}" - ; Automatically close when finished + # Automatically close when finished SetAutoClose true - ; Prompt to remove the Installation directory + # Prompt to remove the Installation directory ${IfNot} $DeleteInstallDir == 1 MessageBox MB_ICONQUESTION|MB_YESNO|MB_DEFBUTTON2 \ "Would you like to completely remove $INSTDIR and all of its contents?" \ /SD IDNO IDNO finished ${EndIf} - ; Make sure you're not removing Program Files + # Make sure you're not removing Program Files ${If} $INSTDIR != 'Program Files' ${AndIf} $INSTDIR != 'Program Files (x86)' RMDir /r "$INSTDIR" @@ -526,7 +548,7 @@ FunctionEnd Function Trim - Exch $R1 ; Original string + Exch $R1 # Original string Push $R2 Loop: @@ -558,36 +580,36 @@ Function Trim FunctionEnd -;------------------------------------------------------------------------------ -; StrStr Function -; - find substring in a string -; -; Usage: -; Push "this is some string" -; Push "some" -; Call StrStr -; Pop $0 ; "some string" -;------------------------------------------------------------------------------ +#------------------------------------------------------------------------------ +# StrStr Function +# - find substring in a string +# +# Usage: +# Push "this is some string" +# Push "some" +# Call StrStr +# Pop $0 ; "some string" +#------------------------------------------------------------------------------ !macro StrStr un Function ${un}StrStr - Exch $R1 ; $R1=substring, stack=[old$R1,string,...] - Exch ; stack=[string,old$R1,...] - Exch $R2 ; $R2=string, stack=[old$R2,old$R1,...] - Push $R3 ; $R3=strlen(substring) - Push $R4 ; $R4=count - Push $R5 ; $R5=tmp - StrLen $R3 $R1 ; Get the length of the Search String - StrCpy $R4 0 ; Set the counter to 0 + Exch $R1 # $R1=substring, stack=[old$R1,string,...] + Exch # stack=[string,old$R1,...] + Exch $R2 # $R2=string, stack=[old$R2,old$R1,...] + Push $R3 # $R3=strlen(substring) + Push $R4 # $R4=count + Push $R5 # $R5=tmp + StrLen $R3 $R1 # Get the length of the Search String + StrCpy $R4 0 # Set the counter to 0 loop: - StrCpy $R5 $R2 $R3 $R4 ; Create a moving window of the string that is - ; the size of the length of the search string - StrCmp $R5 $R1 done ; Is the contents of the window the same as - ; search string, then done - StrCmp $R5 "" done ; Is the window empty, then done - IntOp $R4 $R4 + 1 ; Shift the windows one character - Goto loop ; Repeat + StrCpy $R5 $R2 $R3 $R4 # Create a moving window of the string that is + # the size of the length of the search string + StrCmp $R5 $R1 done # Is the contents of the window the same as + # search string, then done + StrCmp $R5 "" done # Is the window empty, then done + IntOp $R4 $R4 + 1 # Shift the windows one character + Goto loop # Repeat done: StrCpy $R1 $R2 "" $R4 @@ -595,7 +617,7 @@ Function ${un}StrStr Pop $R4 Pop $R3 Pop $R2 - Exch $R1 ; $R1=old$R1, stack=[result,...] + Exch $R1 # $R1=old$R1, stack=[result,...] FunctionEnd !macroend @@ -603,74 +625,74 @@ FunctionEnd !insertmacro StrStr "un." -;------------------------------------------------------------------------------ -; AddToPath Function -; - Adds item to Path for All Users -; - Overcomes NSIS ReadRegStr limitation of 1024 characters by using Native -; Windows Commands -; -; Usage: -; Push "C:\path\to\add" -; Call AddToPath -;------------------------------------------------------------------------------ +#------------------------------------------------------------------------------ +# AddToPath Function +# - Adds item to Path for All Users +# - Overcomes NSIS ReadRegStr limitation of 1024 characters by using Native +# Windows Commands +# +# Usage: +# Push "C:\path\to\add" +# Call AddToPath +#------------------------------------------------------------------------------ !define Environ 'HKLM "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"' Function AddToPath - Exch $0 ; Path to add - Push $1 ; Current Path - Push $2 ; Results of StrStr / Length of Path + Path to Add - Push $3 ; Handle to Reg / Length of Path - Push $4 ; Result of Registry Call + Exch $0 # Path to add + Push $1 # Current Path + Push $2 # Results of StrStr / Length of Path + Path to Add + Push $3 # Handle to Reg / Length of Path + Push $4 # Result of Registry Call - ; Open a handle to the key in the registry, handle in $3, Error in $4 + # Open a handle to the key in the registry, handle in $3, Error in $4 System::Call "advapi32::RegOpenKey(i 0x80000002, t'SYSTEM\CurrentControlSet\Control\Session Manager\Environment', *i.r3) i.r4" - ; Make sure registry handle opened successfully (returned 0) + # Make sure registry handle opened successfully (returned 0) IntCmp $4 0 0 done done - ; Load the contents of path into $1, Error Code into $4, Path length into $2 + # Load the contents of path into $1, Error Code into $4, Path length into $2 System::Call "advapi32::RegQueryValueEx(i $3, t'PATH', i 0, i 0, t.r1, *i ${NSIS_MAX_STRLEN} r2) i.r4" - ; Close the handle to the registry ($3) + # Close the handle to the registry ($3) System::Call "advapi32::RegCloseKey(i $3)" - ; Check for Error Code 234, Path too long for the variable - IntCmp $4 234 0 +4 +4 ; $4 == ERROR_MORE_DATA + # Check for Error Code 234, Path too long for the variable + IntCmp $4 234 0 +4 +4 # $4 == ERROR_MORE_DATA DetailPrint "AddToPath Failed: original length $2 > ${NSIS_MAX_STRLEN}" MessageBox MB_OK \ "You may add C:\salt to the %PATH% for convenience when issuing local salt commands from the command line." \ /SD IDOK Goto done - ; If no error, continue - IntCmp $4 0 +5 ; $4 != NO_ERROR - ; Error 2 means the Key was not found - IntCmp $4 2 +3 ; $4 != ERROR_FILE_NOT_FOUND + # If no error, continue + IntCmp $4 0 +5 # $4 != NO_ERROR + # Error 2 means the Key was not found + IntCmp $4 2 +3 # $4 != ERROR_FILE_NOT_FOUND DetailPrint "AddToPath: unexpected error code $4" Goto done StrCpy $1 "" - ; Check if already in PATH - Push "$1;" ; The string to search - Push "$0;" ; The string to find + # Check if already in PATH + Push "$1;" # The string to search + Push "$0;" # The string to find Call StrStr - Pop $2 ; The result of the search - StrCmp $2 "" 0 done ; String not found, try again with ';' at the end - ; Otherwise, it's already in the path - Push "$1;" ; The string to search - Push "$0\;" ; The string to find + Pop $2 # The result of the search + StrCmp $2 "" 0 done # String not found, try again with ';' at the end + # Otherwise, it's already in the path + Push "$1;" # The string to search + Push "$0\;" # The string to find Call StrStr - Pop $2 ; The result - StrCmp $2 "" 0 done ; String not found, continue (add) - ; Otherwise, it's already in the path + Pop $2 # The result + StrCmp $2 "" 0 done # String not found, continue (add) + # Otherwise, it's already in the path - ; Prevent NSIS string overflow - StrLen $2 $0 ; Length of path to add ($2) - StrLen $3 $1 ; Length of current path ($3) - IntOp $2 $2 + $3 ; Length of current path + path to add ($2) - IntOp $2 $2 + 2 ; Account for the additional ';' - ; $2 = strlen(dir) + strlen(PATH) + sizeof(";") + # Prevent NSIS string overflow + StrLen $2 $0 # Length of path to add ($2) + StrLen $3 $1 # Length of current path ($3) + IntOp $2 $2 + $3 # Length of current path + path to add ($2) + IntOp $2 $2 + 2 # Account for the additional ';' + # $2 = strlen(dir) + strlen(PATH) + sizeof(";") - ; Make sure the new length isn't over the NSIS_MAX_STRLEN + # Make sure the new length isn't over the NSIS_MAX_STRLEN IntCmp $2 ${NSIS_MAX_STRLEN} +4 +4 0 DetailPrint "AddToPath: new length $2 > ${NSIS_MAX_STRLEN}" MessageBox MB_OK \ @@ -678,18 +700,18 @@ Function AddToPath /SD IDOK Goto done - ; Append dir to PATH + # Append dir to PATH DetailPrint "Add to PATH: $0" - StrCpy $2 $1 1 -1 ; Copy the last character of the existing path - StrCmp $2 ";" 0 +2 ; Check for trailing ';' - StrCpy $1 $1 -1 ; remove trailing ';' - StrCmp $1 "" +2 ; Make sure Path is not empty - StrCpy $0 "$1;$0" ; Append new path at the end ($0) + StrCpy $2 $1 1 -1 # Copy the last character of the existing path + StrCmp $2 ";" 0 +2 # Check for trailing ';' + StrCpy $1 $1 -1 # remove trailing ';' + StrCmp $1 "" +2 # Make sure Path is not empty + StrCpy $0 "$1;$0" # Append new path at the end ($0) - ; We can use the NSIS command here. Only 'ReadRegStr' is affected + # We can use the NSIS command here. Only 'ReadRegStr' is affected WriteRegExpandStr ${Environ} "PATH" $0 - ; Broadcast registry change to open programs + # Broadcast registry change to open programs SendMessage ${HWND_BROADCAST} ${WM_WININICHANGE} 0 "STR:Environment" /TIMEOUT=5000 done: @@ -702,16 +724,16 @@ Function AddToPath FunctionEnd -;------------------------------------------------------------------------------ -; RemoveFromPath Function -; - Removes item from Path for All Users -; - Overcomes NSIS ReadRegStr limitation of 1024 characters by using Native -; Windows Commands -; -; Usage: -; Push "C:\path\to\add" -; Call un.RemoveFromPath -;------------------------------------------------------------------------------ +#------------------------------------------------------------------------------ +# RemoveFromPath Function +# - Removes item from Path for All Users +# - Overcomes NSIS ReadRegStr limitation of 1024 characters by using Native +# Windows Commands +# +# Usage: +# Push "C:\path\to\add" +# Call un.RemoveFromPath +#------------------------------------------------------------------------------ Function un.RemoveFromPath Exch $0 @@ -722,59 +744,59 @@ Function un.RemoveFromPath Push $5 Push $6 - ; Open a handle to the key in the registry, handle in $3, Error in $4 + # Open a handle to the key in the registry, handle in $3, Error in $4 System::Call "advapi32::RegOpenKey(i 0x80000002, t'SYSTEM\CurrentControlSet\Control\Session Manager\Environment', *i.r3) i.r4" - ; Make sure registry handle opened successfully (returned 0) + # Make sure registry handle opened successfully (returned 0) IntCmp $4 0 0 done done - ; Load the contents of path into $1, Error Code into $4, Path length into $2 + # Load the contents of path into $1, Error Code into $4, Path length into $2 System::Call "advapi32::RegQueryValueEx(i $3, t'PATH', i 0, i 0, t.r1, *i ${NSIS_MAX_STRLEN} r2) i.r4" - ; Close the handle to the registry ($3) + # Close the handle to the registry ($3) System::Call "advapi32::RegCloseKey(i $3)" - ; Check for Error Code 234, Path too long for the variable - IntCmp $4 234 0 +4 +4 ; $4 == ERROR_MORE_DATA + # Check for Error Code 234, Path too long for the variable + IntCmp $4 234 0 +4 +4 # $4 == ERROR_MORE_DATA DetailPrint "AddToPath: original length $2 > ${NSIS_MAX_STRLEN}" Goto done - ; If no error, continue - IntCmp $4 0 +5 ; $4 != NO_ERROR - ; Error 2 means the Key was not found - IntCmp $4 2 +3 ; $4 != ERROR_FILE_NOT_FOUND + # If no error, continue + IntCmp $4 0 +5 # $4 != NO_ERROR + # Error 2 means the Key was not found + IntCmp $4 2 +3 # $4 != ERROR_FILE_NOT_FOUND DetailPrint "AddToPath: unexpected error code $4" Goto done StrCpy $1 "" - ; Ensure there's a trailing ';' - StrCpy $5 $1 1 -1 ; Copy the last character of the path - StrCmp $5 ";" +2 ; Check for trailing ';', if found continue - StrCpy $1 "$1;" ; ensure trailing ';' + # Ensure there's a trailing ';' + StrCpy $5 $1 1 -1 # Copy the last character of the path + StrCmp $5 ";" +2 # Check for trailing ';', if found continue + StrCpy $1 "$1;" # ensure trailing ';' - ; Check for our directory inside the path - Push $1 ; String to Search - Push "$0;" ; Dir to Find + # Check for our directory inside the path + Push $1 # String to Search + Push "$0;" # Dir to Find Call un.StrStr - Pop $2 ; The results of the search - StrCmp $2 "" done ; If results are empty, we're done, otherwise continue + Pop $2 # The results of the search + StrCmp $2 "" done # If results are empty, we're done, otherwise continue - ; Remove our Directory from the Path + # Remove our Directory from the Path DetailPrint "Remove from PATH: $0" - StrLen $3 "$0;" ; Get the length of our dir ($3) - StrLen $4 $2 ; Get the length of the return from StrStr ($4) - StrCpy $5 $1 -$4 ; $5 is now the part before the path to remove - StrCpy $6 $2 "" $3 ; $6 is now the part after the path to remove - StrCpy $3 "$5$6" ; Combine $5 and $6 + StrLen $3 "$0;" # Get the length of our dir ($3) + StrLen $4 $2 # Get the length of the return from StrStr ($4) + StrCpy $5 $1 -$4 # $5 is now the part before the path to remove + StrCpy $6 $2 "" $3 # $6 is now the part after the path to remove + StrCpy $3 "$5$6" # Combine $5 and $6 - ; Check for Trailing ';' - StrCpy $5 $3 1 -1 ; Load the last character of the string - StrCmp $5 ";" 0 +2 ; Check for ';' - StrCpy $3 $3 -1 ; remove trailing ';' + # Check for Trailing ';' + StrCpy $5 $3 1 -1 # Load the last character of the string + StrCmp $5 ";" 0 +2 # Check for ';' + StrCpy $3 $3 -1 # remove trailing ';' - ; Write the new path to the registry + # Write the new path to the registry WriteRegExpandStr ${Environ} "PATH" $3 - ; Broadcast the change to all open applications + # Broadcast the change to all open applications SendMessage ${HWND_BROADCAST} ${WM_WININICHANGE} 0 "STR:Environment" /TIMEOUT=5000 done: @@ -808,6 +830,7 @@ Function getMinionConfig confFound: FileOpen $0 "$INSTDIR\conf\minion" r + ClearErrors confLoop: FileRead $0 $1 IfErrors EndOfFile @@ -838,68 +861,69 @@ FunctionEnd Function updateMinionConfig ClearErrors - FileOpen $0 "$INSTDIR\conf\minion" "r" ; open target file for reading - GetTempFileName $R0 ; get new temp file name - FileOpen $1 $R0 "w" ; open temp file for writing + FileOpen $0 "$INSTDIR\conf\minion" "r" # open target file for reading + GetTempFileName $R0 # get new temp file name + FileOpen $1 $R0 "w" # open temp file for writing - loop: ; loop through each line - FileRead $0 $2 ; read line from target file - IfErrors done ; end if errors are encountered (end of line) + loop: # loop through each line + FileRead $0 $2 # read line from target file + IfErrors done # end if errors are encountered (end of line) - ${If} $MasterHost_State != "" ; if master is empty - ${AndIf} $MasterHost_State != "salt" ; and if master is not 'salt' - ${StrLoc} $3 $2 "master:" ">" ; where is 'master:' in this line - ${If} $3 == 0 ; is it in the first... - ${OrIf} $3 == 1 ; or second position (account for comments) - StrCpy $2 "master: $MasterHost_State$\r$\n" ; write the master - ${EndIf} ; close if statement - ${EndIf} ; close if statement + ${If} $MasterHost_State != "" # if master is empty + ${AndIf} $MasterHost_State != "salt" # and if master is not 'salt' + ${StrLoc} $3 $2 "master:" ">" # where is 'master:' in this line + ${If} $3 == 0 # is it in the first... + ${OrIf} $3 == 1 # or second position (account for comments) + StrCpy $2 "master: $MasterHost_State$\r$\n" # write the master + ${EndIf} # close if statement + ${EndIf} # close if statement - ${If} $MinionName_State != "" ; if minion is empty - ${AndIf} $MinionName_State != "hostname" ; and if minion is not 'hostname' - ${StrLoc} $3 $2 "id:" ">" ; where is 'id:' in this line - ${If} $3 == 0 ; is it in the first... - ${OrIf} $3 == 1 ; or the second position (account for comments) - StrCpy $2 "id: $MinionName_State$\r$\n" ; change line - ${EndIf} ; close if statement - ${EndIf} ; close if statement + ${If} $MinionName_State != "" # if minion is empty + ${AndIf} $MinionName_State != "hostname" # and if minion is not 'hostname' + ${StrLoc} $3 $2 "id:" ">" # where is 'id:' in this line + ${If} $3 == 0 # is it in the first... + ${OrIf} $3 == 1 # or the second position (account for comments) + StrCpy $2 "id: $MinionName_State$\r$\n" # change line + ${EndIf} # close if statement + ${EndIf} # close if statement - FileWrite $1 $2 ; write changed or unchanged line to temp file + FileWrite $1 $2 # write changed or unchanged line to temp file Goto loop done: - FileClose $0 ; close target file - FileClose $1 ; close temp file - Delete "$INSTDIR\conf\minion" ; delete target file - CopyFiles /SILENT $R0 "$INSTDIR\conf\minion" ; copy temp file to target file - Delete $R0 ; delete temp file + FileClose $0 # close target file + FileClose $1 # close temp file + Delete "$INSTDIR\conf\minion" # delete target file + CopyFiles /SILENT $R0 "$INSTDIR\conf\minion" # copy temp file to target file + Delete $R0 # delete temp file FunctionEnd Function parseCommandLineSwitches - ; Load the parameters + # Load the parameters ${GetParameters} $R0 - ; Check for start-minion switches - ; /start-service is to be deprecated, so we must check for both + # Check for start-minion switches + # /start-service is to be deprecated, so we must check for both ${GetOptions} $R0 "/start-service=" $R1 ${GetOptions} $R0 "/start-minion=" $R2 # Service: Start Salt Minion ${IfNot} $R2 == "" - ; If start-minion was passed something, then set it + # If start-minion was passed something, then set it StrCpy $StartMinion $R2 ${ElseIfNot} $R1 == "" - ; If start-service was passed something, then set StartMinion to that + # If start-service was passed something, then set StartMinion to that StrCpy $StartMinion $R1 ${Else} - ; Otherwise default to 1 + # Otherwise default to 1 StrCpy $StartMinion 1 ${EndIf} # Service: Minion Startup Type Delayed + ClearErrors ${GetOptions} $R0 "/start-minion-delayed" $R1 IfErrors start_minion_delayed_not_found StrCpy $StartMinionDelayed 1 diff --git a/salt/config/__init__.py b/salt/config/__init__.py index c0ba79f1cb2..74dbf1a6c72 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -343,7 +343,7 @@ VALID_OPTS = { # Whether or not scheduled mine updates should be accompanied by a job return for the job cache 'mine_return_job': bool, - # Schedule a mine update every n number of seconds + # The number of minutes between mine updates. 'mine_interval': int, # The ipc strategy. (i.e., sockets versus tcp, etc) @@ -590,6 +590,23 @@ VALID_OPTS = { # False in 2016.3.0 'add_proxymodule_to_opts': bool, + # Merge pillar data into configuration opts. + # As multiple proxies can run on the same server, we may need different + # configuration options for each, while there's one single configuration file. + # The solution is merging the pillar data of each proxy minion into the opts. + 'proxy_merge_pillar_in_opts': bool, + + # Deep merge of pillar data into configuration opts. + # Evaluated only when `proxy_merge_pillar_in_opts` is True. + 'proxy_deep_merge_pillar_in_opts': bool, + + # The strategy used when merging pillar into opts. + # Considered only when `proxy_merge_pillar_in_opts` is True. + 'proxy_merge_pillar_in_opts_strategy': str, + + # Allow enabling mine details using pillar data. + 'proxy_mines_pillar': bool, + # In some particular cases, always alive proxies are not beneficial. # This option can be used in those less dynamic environments: # the user can request the connection @@ -1700,6 +1717,12 @@ DEFAULT_PROXY_MINION_OPTS = { 'append_minionid_config_dirs': ['cachedir', 'pidfile', 'default_include', 'extension_modules'], 'default_include': 'proxy.d/*.conf', + 'proxy_merge_pillar_in_opts': False, + 'proxy_deep_merge_pillar_in_opts': False, + 'proxy_merge_pillar_in_opts_strategy': 'smart', + + 'proxy_mines_pillar': True, + # By default, proxies will preserve the connection. # If this option is set to False, # the connection with the remote dumb device diff --git a/salt/fileclient.py b/salt/fileclient.py index eaaf7ddb91c..cb3b210a037 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -1295,10 +1295,10 @@ class RemoteClient(Client): hash_type = self.opts.get(u'hash_type', u'md5') ret[u'hsum'] = salt.utils.get_hash(path, form=hash_type) ret[u'hash_type'] = hash_type - return ret, list(os.stat(path)) + return ret load = {u'path': path, u'saltenv': saltenv, - u'cmd': u'_file_hash_and_stat'} + u'cmd': u'_file_hash'} return self.channel.send(load) def hash_file(self, path, saltenv=u'base'): @@ -1307,14 +1307,33 @@ class RemoteClient(Client): master file server prepend the path with salt:// otherwise, prepend the file with / for a local file. ''' - return self.__hash_and_stat_file(path, saltenv)[0] + return self.__hash_and_stat_file(path, saltenv) def hash_and_stat_file(self, path, saltenv=u'base'): ''' The same as hash_file, but also return the file's mode, or None if no mode data is present. ''' - return self.__hash_and_stat_file(path, saltenv) + hash_result = self.hash_file(path, saltenv) + try: + path = self._check_proto(path) + except MinionError as err: + if not os.path.isfile(path): + return hash_result, None + else: + try: + return hash_result, list(os.stat(path)) + except Exception: + return hash_result, None + load = {'path': path, + 'saltenv': saltenv, + 'cmd': '_file_find'} + fnd = self.channel.send(load) + try: + stat_result = fnd.get('stat') + except AttributeError: + stat_result = None + return hash_result, stat_result def list_env(self, saltenv=u'base'): ''' diff --git a/salt/loader.py b/salt/loader.py index 9399437fee7..81e44ebb27a 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -270,7 +270,7 @@ def raw_mod(opts, name, functions, mod=u'modules'): testmod['test.ping']() ''' loader = LazyLoader( - _module_dirs(opts, mod, u'rawmodule'), + _module_dirs(opts, mod, u'module'), opts, tag=u'rawmodule', virtual_enable=False, diff --git a/salt/minion.py b/salt/minion.py index dc07817dc2e..d51445be28a 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -103,6 +103,7 @@ import salt.defaults.exitcodes import salt.cli.daemons import salt.log.setup +import salt.utils.dictupdate from salt.config import DEFAULT_MINION_OPTS from salt.defaults import DEFAULT_TARGET_DELIM from salt.utils.debug import enable_sigusr1_handler @@ -3213,6 +3214,26 @@ class ProxyMinion(Minion): if u'proxy' not in self.opts: self.opts[u'proxy'] = self.opts[u'pillar'][u'proxy'] + if self.opts.get(u'proxy_merge_pillar_in_opts'): + # Override proxy opts with pillar data when the user required. + self.opts = salt.utils.dictupdate.merge(self.opts, + self.opts[u'pillar'], + strategy=self.opts.get(u'proxy_merge_pillar_in_opts_strategy'), + merge_lists=self.opts.get(u'proxy_deep_merge_pillar_in_opts', False)) + elif self.opts.get(u'proxy_mines_pillar'): + # Even when not required, some details such as mine configuration + # should be merged anyway whenever possible. + if u'mine_interval' in self.opts[u'pillar']: + self.opts[u'mine_interval'] = self.opts[u'pillar'][u'mine_interval'] + if u'mine_functions' in self.opts[u'pillar']: + general_proxy_mines = self.opts.get(u'mine_functions', []) + specific_proxy_mines = self.opts[u'pillar'][u'mine_functions'] + try: + self.opts[u'mine_functions'] = general_proxy_mines + specific_proxy_mines + except TypeError as terr: + log.error(u'Unable to merge mine functions from the pillar in the opts, for proxy {}'.format( + self.opts[u'id'])) + fq_proxyname = self.opts[u'proxy'][u'proxytype'] # Need to load the modules so they get all the dunder variables diff --git a/salt/modules/alternatives.py b/salt/modules/alternatives.py index b1a5ecb4a81..ad21785fc39 100644 --- a/salt/modules/alternatives.py +++ b/salt/modules/alternatives.py @@ -12,6 +12,7 @@ import logging # Import Salt libs import salt.utils.files +import salt.utils.path # Import 3rd-party libs from salt.ext import six @@ -241,4 +242,4 @@ def _read_link(name): Throws an OSError if the link does not exist ''' alt_link_path = '/etc/alternatives/{0}'.format(name) - return os.readlink(alt_link_path) + return salt.utils.path.readlink(alt_link_path) diff --git a/salt/modules/debian_ip.py b/salt/modules/debian_ip.py index 60b60f893ce..6b5c760c924 100644 --- a/salt/modules/debian_ip.py +++ b/salt/modules/debian_ip.py @@ -2038,19 +2038,12 @@ def build_network_settings(**settings): # Write settings _write_file_network(network, _DEB_NETWORKING_FILE, True) - # Write hostname to /etc/hostname + # Get hostname and domain from opts sline = opts['hostname'].split('.', 1) opts['hostname'] = sline[0] - hostname = '{0}\n' . format(opts['hostname']) current_domainname = current_network_settings['domainname'] current_searchdomain = current_network_settings['searchdomain'] - # Only write the hostname if it has changed - if not opts['hostname'] == current_network_settings['hostname']: - if not ('test' in settings and settings['test']): - # TODO replace wiht a call to network.mod_hostname instead - _write_file_network(hostname, _DEB_HOSTNAME_FILE) - new_domain = False if len(sline) > 1: new_domainname = sline[1] diff --git a/salt/modules/genesis.py b/salt/modules/genesis.py index 32d5f461b52..ab0eed51384 100644 --- a/salt/modules/genesis.py +++ b/salt/modules/genesis.py @@ -17,11 +17,12 @@ except ImportError: from pipes import quote as _cmd_quote # Import salt libs -import salt.utils.path -import salt.utils.yast -import salt.utils.preseed -import salt.utils.kickstart import salt.syspaths +import salt.utils.kickstart +import salt.utils.path +import salt.utils.preseed +import salt.utils.validate.path +import salt.utils.yast from salt.exceptions import SaltInvocationError # Import 3rd-party libs @@ -399,10 +400,15 @@ def _bootstrap_deb( if repo_url is None: repo_url = 'http://ftp.debian.org/debian/' - if not salt.utils.which('debootstrap'): + if not salt.utils.path.which('debootstrap'): log.error('Required tool debootstrap is not installed.') return False + if static_qemu and not salt.utils.validate.path.is_executable(static_qemu): + log.error('Required tool qemu not ' + 'present/readable at: {0}'.format(static_qemu)) + return False + if isinstance(pkgs, (list, tuple)): pkgs = ','.join(pkgs) if isinstance(exclude_pkgs, (list, tuple)): @@ -427,11 +433,13 @@ def _bootstrap_deb( __salt__['cmd.run'](deb_args, python_shell=False) - __salt__['cmd.run']( - 'cp {qemu} {root}/usr/bin/'.format( - qemu=_cmd_quote(static_qemu), root=_cmd_quote(root) + if static_qemu: + __salt__['cmd.run']( + 'cp {qemu} {root}/usr/bin/'.format( + qemu=_cmd_quote(static_qemu), root=_cmd_quote(root) + ) ) - ) + env = {'DEBIAN_FRONTEND': 'noninteractive', 'DEBCONF_NONINTERACTIVE_SEEN': 'true', 'LC_ALL': 'C', diff --git a/salt/modules/groupadd.py b/salt/modules/groupadd.py index 2f79c47dd90..f02a5811a7b 100644 --- a/salt/modules/groupadd.py +++ b/salt/modules/groupadd.py @@ -31,7 +31,7 @@ def __virtual__(): if __grains__['kernel'] in ('Linux', 'OpenBSD', 'NetBSD'): return __virtualname__ return (False, 'The groupadd execution module cannot be loaded: ' - ' only available on Linux, OpenBSD and NetBSD') + ' only available on Linux, OpenBSD and NetBSD') def add(name, gid=None, system=False, root=None): @@ -44,12 +44,12 @@ def add(name, gid=None, system=False, root=None): salt '*' group.add foo 3456 ''' - cmd = 'groupadd ' + cmd = ['groupadd'] if gid: - cmd += '-g {0} '.format(gid) + cmd.append('-g {0}'.format(gid)) if system and __grains__['kernel'] != 'OpenBSD': - cmd += '-r ' - cmd += name + cmd.append('-r') + cmd.append(name) if root is not None: cmd.extend(('-R', root)) @@ -69,7 +69,7 @@ def delete(name, root=None): salt '*' group.delete foo ''' - cmd = ('groupdel', name) + cmd = ['groupdel', name] if root is not None: cmd.extend(('-R', root)) @@ -140,7 +140,7 @@ def chgid(name, gid, root=None): pre_gid = __salt__['file.group_to_gid'](name) if gid == pre_gid: return True - cmd = ('groupmod', '-g', gid, name) + cmd = ['groupmod', '-g', gid, name] if root is not None: cmd.extend(('-R', root)) @@ -170,15 +170,15 @@ def adduser(name, username, root=None): if __grains__['kernel'] == 'Linux': if on_redhat_5: - cmd = ('gpasswd', '-a', username, name) + cmd = ['gpasswd', '-a', username, name] elif on_suse_11: - cmd = ('usermod', '-A', name, username) + cmd = ['usermod', '-A', name, username] else: - cmd = ('gpasswd', '--add', username, name) + cmd = ['gpasswd', '--add', username, name] if root is not None: cmd.extend(('-Q', root)) else: - cmd = ('usermod', '-G', name, username) + cmd = ['usermod', '-G', name, username] if root is not None: cmd.extend(('-R', root)) @@ -208,20 +208,20 @@ def deluser(name, username, root=None): if username in grp_info['members']: if __grains__['kernel'] == 'Linux': if on_redhat_5: - cmd = ('gpasswd', '-d', username, name) + cmd = ['gpasswd', '-d', username, name] elif on_suse_11: - cmd = ('usermod', '-R', name, username) + cmd = ['usermod', '-R', name, username] else: - cmd = ('gpasswd', '--del', username, name) + cmd = ['gpasswd', '--del', username, name] if root is not None: cmd.extend(('-R', root)) retcode = __salt__['cmd.retcode'](cmd, python_shell=False) elif __grains__['kernel'] == 'OpenBSD': out = __salt__['cmd.run_stdout']('id -Gn {0}'.format(username), python_shell=False) - cmd = 'usermod -S ' - cmd += ','.join([g for g in out.split() if g != str(name)]) - cmd += ' {0}'.format(username) + cmd = ['usermod', '-S'] + cmd.append(','.join([g for g in out.split() if g != str(name)])) + cmd.append('{0}'.format(username)) retcode = __salt__['cmd.retcode'](cmd, python_shell=False) else: log.error('group.deluser is not yet supported on this platform') @@ -249,13 +249,13 @@ def members(name, members_list, root=None): if __grains__['kernel'] == 'Linux': if on_redhat_5: - cmd = ('gpasswd', '-M', members_list, name) + cmd = ['gpasswd', '-M', members_list, name] elif on_suse_11: for old_member in __salt__['group.info'](name).get('members'): __salt__['cmd.run']('groupmod -R {0} {1}'.format(old_member, name), python_shell=False) - cmd = ('groupmod', '-A', members_list, name) + cmd = ['groupmod', '-A', members_list, name] else: - cmd = ('gpasswd', '--members', members_list, name) + cmd = ['gpasswd', '--members', members_list, name] if root is not None: cmd.extend(('-R', root)) retcode = __salt__['cmd.retcode'](cmd, python_shell=False) @@ -270,7 +270,7 @@ def members(name, members_list, root=None): for user in members_list.split(","): if user: retcode = __salt__['cmd.retcode']( - 'usermod -G {0} {1}'.format(name, user), + ['usermod', '-G', name, user], python_shell=False) if not retcode == 0: break diff --git a/salt/modules/ini_manage.py b/salt/modules/ini_manage.py index 5b74e5a1b43..9af24eda199 100644 --- a/salt/modules/ini_manage.py +++ b/salt/modules/ini_manage.py @@ -318,17 +318,18 @@ class _Section(OrderedDict): yield '{0}[{1}]{0}'.format(os.linesep, self.name) sections_dict = OrderedDict() for name, value in six.iteritems(self): + # Handle Comment Lines if com_regx.match(name): yield '{0}{1}'.format(value, os.linesep) + # Handle Sections elif isinstance(value, _Section): sections_dict.update({name: value}) + # Key / Value pairs + # Adds spaces between the separator else: yield '{0}{1}{2}{3}'.format( name, - ( - ' {0} '.format(self.sep) if self.sep != ' ' - else self.sep - ), + ' {0} '.format(self.sep) if self.sep != ' ' else self.sep, value, os.linesep ) diff --git a/salt/modules/redismod.py b/salt/modules/redismod.py index ee15a45a03c..a95e1b9f3f4 100644 --- a/salt/modules/redismod.py +++ b/salt/modules/redismod.py @@ -18,6 +18,8 @@ Module to provide redis functionality to Salt # Import Python libs from __future__ import absolute_import from salt.ext.six.moves import zip +from salt.ext import six +from datetime import datetime # Import third party libs try: @@ -513,8 +515,14 @@ def lastsave(host=None, port=None, db=None, password=None): salt '*' redis.lastsave ''' + # Use of %s to get the timestamp is not supported by Python. The reason it + # works is because it's passed to the system strftime which may not support + # it. See: https://stackoverflow.com/a/11743262 server = _connect(host, port, db, password) - return int(server.lastsave().strftime("%s")) + if six.PY2: + return int((server.lastsave() - datetime(1970, 1, 1)).total_seconds()) + else: + return int(server.lastsave().timestamp()) def llen(key, host=None, port=None, db=None, password=None): diff --git a/salt/modules/virtualenv_mod.py b/salt/modules/virtualenv_mod.py index 8c2849e2ef7..20145c57e96 100644 --- a/salt/modules/virtualenv_mod.py +++ b/salt/modules/virtualenv_mod.py @@ -200,12 +200,10 @@ def create(path, for entry in extra_search_dir: cmd.append('--extra-search-dir={0}'.format(entry)) if never_download is True: - if virtualenv_version_info >= (1, 10): + if virtualenv_version_info >= (1, 10) and virtualenv_version_info < (14, 0, 0): log.info( - 'The virtualenv \'--never-download\' option has been ' - 'deprecated in virtualenv(>=1.10), as such, the ' - '\'never_download\' option to `virtualenv.create()` has ' - 'also been deprecated and it\'s not necessary anymore.' + '--never-download was deprecated in 1.10.0, but reimplemented in 14.0.0. ' + 'If this feature is needed, please install a supported virtualenv version.' ) else: cmd.append('--never-download') diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 842eb943f79..c729e07ef5c 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -987,18 +987,6 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # Version 1.2.3 will apply to packages foo and bar salt '*' pkg.install foo,bar version=1.2.3 - cache_file (str): - A single file to copy down for use with the installer. Copied to the - same location as the installer. Use this over ``cache_dir`` if there - are many files in the directory and you only need a specific file - and don't want to cache additional files that may reside in the - installer directory. Only applies to files on ``salt://`` - - cache_dir (bool): - True will copy the contents of the installer directory. This is - useful for installations that are not a single file. Only applies to - directories on ``salt://`` - extra_install_flags (str): Additional install flags that will be appended to the ``install_flags`` defined in the software definition file. Only @@ -1290,7 +1278,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): if use_msiexec: cmd = msiexec arguments = ['/i', cached_pkg] - if pkginfo['version_num'].get('allusers', True): + if pkginfo[version_num].get('allusers', True): arguments.append('ALLUSERS="1"') arguments.extend(salt.utils.shlex_split(install_flags)) else: diff --git a/salt/returners/local_cache.py b/salt/returners/local_cache.py index dee91a4e8c4..c0b135c3efa 100644 --- a/salt/returners/local_cache.py +++ b/salt/returners/local_cache.py @@ -398,7 +398,6 @@ def clean_old_jobs(): Clean out the old jobs from the job cache ''' if __opts__['keep_jobs'] != 0: - cur = time.time() jid_root = _job_dir() if not os.path.exists(jid_root): @@ -428,7 +427,7 @@ def clean_old_jobs(): shutil.rmtree(t_path) elif os.path.isfile(jid_file): jid_ctime = os.stat(jid_file).st_ctime - hours_difference = (cur - jid_ctime) / 3600.0 + hours_difference = (time.time()- jid_ctime) / 3600.0 if hours_difference > __opts__['keep_jobs'] and os.path.exists(t_path): # Remove the entire t_path from the original JID dir shutil.rmtree(t_path) @@ -442,7 +441,7 @@ def clean_old_jobs(): # Checking the time again prevents a possible race condition where # t_path JID dirs were created, but not yet populated by a jid file. t_path_ctime = os.stat(t_path).st_ctime - hours_difference = (cur - t_path_ctime) / 3600.0 + hours_difference = (time.time() - t_path_ctime) / 3600.0 if hours_difference > __opts__['keep_jobs']: shutil.rmtree(t_path) diff --git a/salt/utils/validate/path.py b/salt/utils/validate/path.py index 1385b9bbce0..87f2d789c7b 100644 --- a/salt/utils/validate/path.py +++ b/salt/utils/validate/path.py @@ -64,3 +64,14 @@ def is_readable(path): # The path does not exist return False + + +def is_executable(path): + ''' + Check if a given path is executable by the current user. + + :param path: The path to check + :returns: True or False + ''' + + return os.access(path, os.X_OK) diff --git a/tests/unit/modules/test_alternatives.py b/tests/unit/modules/test_alternatives.py index b018eafa803..8dd0cde28d6 100644 --- a/tests/unit/modules/test_alternatives.py +++ b/tests/unit/modules/test_alternatives.py @@ -66,30 +66,28 @@ class AlternativesTestCase(TestCase, LoaderModuleMockMixin): ) def test_show_current(self): - with patch('os.readlink') as os_readlink_mock: - os_readlink_mock.return_value = '/etc/alternatives/salt' + mock = MagicMock(return_value='/etc/alternatives/salt') + with patch('salt.utils.path.readlink', mock): ret = alternatives.show_current('better-world') self.assertEqual('/etc/alternatives/salt', ret) - os_readlink_mock.assert_called_once_with( - '/etc/alternatives/better-world' - ) + mock.assert_called_once_with('/etc/alternatives/better-world') with TestsLoggingHandler() as handler: - os_readlink_mock.side_effect = OSError('Hell was not found!!!') + mock.side_effect = OSError('Hell was not found!!!') self.assertFalse(alternatives.show_current('hell')) - os_readlink_mock.assert_called_with('/etc/alternatives/hell') + mock.assert_called_with('/etc/alternatives/hell') self.assertIn('ERROR:alternative: hell does not exist', handler.messages) def test_check_installed(self): - with patch('os.readlink') as os_readlink_mock: - os_readlink_mock.return_value = '/etc/alternatives/salt' + mock = MagicMock(return_value='/etc/alternatives/salt') + with patch('salt.utils.path.readlink', mock): self.assertTrue( alternatives.check_installed( 'better-world', '/etc/alternatives/salt' ) ) - os_readlink_mock.return_value = False + mock.return_value = False self.assertFalse( alternatives.check_installed( 'help', '/etc/alternatives/salt' diff --git a/tests/unit/modules/test_genesis.py b/tests/unit/modules/test_genesis.py index 424220117b2..0ab9eb03d92 100644 --- a/tests/unit/modules/test_genesis.py +++ b/tests/unit/modules/test_genesis.py @@ -94,9 +94,11 @@ class GenesisTestCase(TestCase, LoaderModuleMockMixin): 'cmd.run': MagicMock(), 'disk.blkid': MagicMock(return_value={})}): with patch('salt.modules.genesis.salt.utils.which', return_value=True): - param_set['params'].update(common_parms) - self.assertEqual(genesis.bootstrap(**param_set['params']), None) - genesis.__salt__['cmd.run'].assert_any_call(param_set['cmd'], python_shell=False) + with patch('salt.modules.genesis.salt.utils.validate.path.is_executable', + return_value=True): + param_set['params'].update(common_parms) + self.assertEqual(genesis.bootstrap(**param_set['params']), None) + genesis.__salt__['cmd.run'].assert_any_call(param_set['cmd'], python_shell=False) with patch.object(genesis, '_bootstrap_pacman', return_value='A') as pacman_patch: with patch.dict(genesis.__salt__, {'mount.umount': MagicMock(), diff --git a/tests/unit/modules/test_groupadd.py b/tests/unit/modules/test_groupadd.py index 29dfd15ed7c..9345cd043e3 100644 --- a/tests/unit/modules/test_groupadd.py +++ b/tests/unit/modules/test_groupadd.py @@ -118,16 +118,16 @@ class GroupAddTestCase(TestCase, LoaderModuleMockMixin): ''' os_version_list = [ {'grains': {'kernel': 'Linux', 'os_family': 'RedHat', 'osmajorrelease': '5'}, - 'cmd': ('gpasswd', '-a', 'root', 'test')}, + 'cmd': ['gpasswd', '-a', 'root', 'test']}, {'grains': {'kernel': 'Linux', 'os_family': 'Suse', 'osmajorrelease': '11'}, - 'cmd': ('usermod', '-A', 'test', 'root')}, + 'cmd': ['usermod', '-A', 'test', 'root']}, {'grains': {'kernel': 'Linux'}, - 'cmd': ('gpasswd', '--add', 'root', 'test')}, + 'cmd': ['gpasswd', '--add', 'root', 'test']}, {'grains': {'kernel': 'OTHERKERNEL'}, - 'cmd': ('usermod', '-G', 'test', 'root')}, + 'cmd': ['usermod', '-G', 'test', 'root']}, ] for os_version in os_version_list: @@ -145,16 +145,16 @@ class GroupAddTestCase(TestCase, LoaderModuleMockMixin): ''' os_version_list = [ {'grains': {'kernel': 'Linux', 'os_family': 'RedHat', 'osmajorrelease': '5'}, - 'cmd': ('gpasswd', '-d', 'root', 'test')}, + 'cmd': ['gpasswd', '-d', 'root', 'test']}, {'grains': {'kernel': 'Linux', 'os_family': 'Suse', 'osmajorrelease': '11'}, - 'cmd': ('usermod', '-R', 'test', 'root')}, + 'cmd': ['usermod', '-R', 'test', 'root']}, {'grains': {'kernel': 'Linux'}, - 'cmd': ('gpasswd', '--del', 'root', 'test')}, + 'cmd': ['gpasswd', '--del', 'root', 'test']}, {'grains': {'kernel': 'OpenBSD'}, - 'cmd': 'usermod -S foo root'}, + 'cmd': ['usermod', '-S', 'foo', 'root']}, ] for os_version in os_version_list: @@ -180,16 +180,16 @@ class GroupAddTestCase(TestCase, LoaderModuleMockMixin): ''' os_version_list = [ {'grains': {'kernel': 'Linux', 'os_family': 'RedHat', 'osmajorrelease': '5'}, - 'cmd': ('gpasswd', '-M', 'foo', 'test')}, + 'cmd': ['gpasswd', '-M', 'foo', 'test']}, {'grains': {'kernel': 'Linux', 'os_family': 'Suse', 'osmajorrelease': '11'}, - 'cmd': ('groupmod', '-A', 'foo', 'test')}, + 'cmd': ['groupmod', '-A', 'foo', 'test']}, {'grains': {'kernel': 'Linux'}, - 'cmd': ('gpasswd', '--members', 'foo', 'test')}, + 'cmd': ['gpasswd', '--members', 'foo', 'test']}, {'grains': {'kernel': 'OpenBSD'}, - 'cmd': 'usermod -G test foo'}, + 'cmd': ['usermod', '-G', 'test', 'foo']}, ] for os_version in os_version_list: diff --git a/tests/unit/modules/test_hosts.py b/tests/unit/modules/test_hosts.py index c7b1b4e9882..56f01f56ab2 100644 --- a/tests/unit/modules/test_hosts.py +++ b/tests/unit/modules/test_hosts.py @@ -16,6 +16,7 @@ from tests.support.mock import ( ) # Import Salt Libs import salt.modules.hosts as hosts +import salt.utils from salt.ext.six.moves import StringIO @@ -92,8 +93,12 @@ class HostsTestCase(TestCase, LoaderModuleMockMixin): ''' Tests true if the alias is set ''' + hosts_file = '/etc/hosts' + if salt.utils.is_windows(): + hosts_file = r'C:\Windows\System32\Drivers\etc\hosts' + with patch('salt.modules.hosts.__get_hosts_filename', - MagicMock(return_value='/etc/hosts')), \ + MagicMock(return_value=hosts_file)), \ patch('os.path.isfile', MagicMock(return_value=False)), \ patch.dict(hosts.__salt__, {'config.option': MagicMock(return_value=None)}): @@ -139,7 +144,16 @@ class HostsTestCase(TestCase, LoaderModuleMockMixin): self.close() def close(self): - data[0] = self.getvalue() + # Don't save unless there's something there. In Windows + # the class gets initialized the first time with mode = w + # which sets the initial value to ''. When the class closes + # it clears out data and causes the test to fail. + # I don't know why it get's initialized with a mode of 'w' + # For the purposes of this test data shouldn't be empty + # This is a problem with this class and not with the hosts + # module + if self.getvalue(): + data[0] = self.getvalue() StringIO.close(self) expected = '\n'.join(( @@ -151,6 +165,7 @@ class HostsTestCase(TestCase, LoaderModuleMockMixin): mock_opt = MagicMock(return_value=None) with patch.dict(hosts.__salt__, {'config.option': mock_opt}): self.assertTrue(hosts.set_host('1.1.1.1', ' ')) + self.assertEqual(data[0], expected) # 'rm_host' function tests: 2 @@ -182,9 +197,13 @@ class HostsTestCase(TestCase, LoaderModuleMockMixin): ''' Tests if specified host entry gets added from the hosts file ''' + hosts_file = '/etc/hosts' + if salt.utils.is_windows(): + hosts_file = r'C:\Windows\System32\Drivers\etc\hosts' + with patch('salt.utils.files.fopen', mock_open()), \ patch('salt.modules.hosts.__get_hosts_filename', - MagicMock(return_value='/etc/hosts')): + MagicMock(return_value=hosts_file)): mock_opt = MagicMock(return_value=None) with patch.dict(hosts.__salt__, {'config.option': mock_opt}): self.assertTrue(hosts.add_host('10.10.10.10', 'Salt1')) diff --git a/tests/unit/modules/test_ini_manage.py b/tests/unit/modules/test_ini_manage.py index 9e41ecae5a7..f3550262a97 100644 --- a/tests/unit/modules/test_ini_manage.py +++ b/tests/unit/modules/test_ini_manage.py @@ -15,38 +15,38 @@ import salt.modules.ini_manage as ini class IniManageTestCase(TestCase): - TEST_FILE_CONTENT = '''\ -# Comment on the first line - -# First main option -option1=main1 - -# Second main option -option2=main2 - - -[main] -# Another comment -test1=value 1 - -test2=value 2 - -[SectionB] -test1=value 1B - -# Blank line should be above -test3 = value 3B - -[SectionC] -# The following option is empty -empty_option= -''' + TEST_FILE_CONTENT = os.linesep.join([ + '# Comment on the first line', + '', + '# First main option', + 'option1=main1', + '', + '# Second main option', + 'option2=main2', + '', + '', + '[main]', + '# Another comment', + 'test1=value 1', + '', + 'test2=value 2', + '', + '[SectionB]', + 'test1=value 1B', + '', + '# Blank line should be above', + 'test3 = value 3B', + '', + '[SectionC]', + '# The following option is empty', + 'empty_option=' + ]) maxDiff = None def setUp(self): - self.tfile = tempfile.NamedTemporaryFile(delete=False, mode='w+') - self.tfile.write(self.TEST_FILE_CONTENT) + self.tfile = tempfile.NamedTemporaryFile(delete=False, mode='w+b') + self.tfile.write(salt.utils.to_bytes(self.TEST_FILE_CONTENT)) self.tfile.close() def tearDown(self): @@ -121,40 +121,42 @@ empty_option= }) with salt.utils.files.fopen(self.tfile.name, 'r') as fp: file_content = fp.read() - self.assertIn('\nempty_option = \n', file_content, - 'empty_option was not preserved') + expected = '{0}{1}{0}'.format(os.linesep, 'empty_option = ') + self.assertIn(expected, file_content, 'empty_option was not preserved') def test_empty_lines_preserved_after_edit(self): ini.set_option(self.tfile.name, { 'SectionB': {'test3': 'new value 3B'}, }) + expected = os.linesep.join([ + '# Comment on the first line', + '', + '# First main option', + 'option1 = main1', + '', + '# Second main option', + 'option2 = main2', + '', + '[main]', + '# Another comment', + 'test1 = value 1', + '', + 'test2 = value 2', + '', + '[SectionB]', + 'test1 = value 1B', + '', + '# Blank line should be above', + 'test3 = new value 3B', + '', + '[SectionC]', + '# The following option is empty', + 'empty_option = ', + '' + ]) with salt.utils.files.fopen(self.tfile.name, 'r') as fp: file_content = fp.read() - self.assertEqual('''\ -# Comment on the first line - -# First main option -option1 = main1 - -# Second main option -option2 = main2 - -[main] -# Another comment -test1 = value 1 - -test2 = value 2 - -[SectionB] -test1 = value 1B - -# Blank line should be above -test3 = new value 3B - -[SectionC] -# The following option is empty -empty_option = -''', file_content) + self.assertEqual(expected, file_content) def test_empty_lines_preserved_after_multiple_edits(self): ini.set_option(self.tfile.name, { diff --git a/tests/unit/modules/test_virtualenv.py b/tests/unit/modules/test_virtualenv.py index 25f511ef1b1..7e86f2bcad7 100644 --- a/tests/unit/modules/test_virtualenv.py +++ b/tests/unit/modules/test_virtualenv.py @@ -109,10 +109,9 @@ class VirtualenvTestCase(TestCase, LoaderModuleMockMixin): # Are we logging the deprecation information? self.assertIn( - 'INFO:The virtualenv \'--never-download\' option has been ' - 'deprecated in virtualenv(>=1.10), as such, the ' - '\'never_download\' option to `virtualenv.create()` has ' - 'also been deprecated and it\'s not necessary anymore.', + 'INFO:--never-download was deprecated in 1.10.0, ' + 'but reimplemented in 14.0.0. If this feature is needed, ' + 'please install a supported virtualenv version.', handler.messages ) diff --git a/tests/unit/returners/test_local_cache.py b/tests/unit/returners/test_local_cache.py index ad9ff53e07b..741957ffd87 100644 --- a/tests/unit/returners/test_local_cache.py +++ b/tests/unit/returners/test_local_cache.py @@ -97,7 +97,10 @@ class LocalCacheCleanOldJobsTestCase(TestCase, LoaderModuleMockMixin): local_cache.clean_old_jobs() # Get the name of the JID directory that was created to test against - jid_dir_name = jid_dir.rpartition('/')[2] + if salt.utils.is_windows(): + jid_dir_name = jid_dir.rpartition('\\')[2] + else: + jid_dir_name = jid_dir.rpartition('/')[2] # Assert the JID directory is still present to be cleaned after keep_jobs interval self.assertEqual([jid_dir_name], os.listdir(TMP_JID_DIR))